npm - @letsscrapedata/scraper - Versions diffs - 0.0.85 → 0.0.87 - Mend

@letsscrapedata/scraper 0.0.85 → 0.0.87

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (4) hide show

package/dist/index.cjs CHANGED Viewed

	@@ -1 +1 @@
1	- "use strict";var __create=Object.create,__defProp=Object.defineProperty,__getOwnPropDesc=Object.getOwnPropertyDescriptor,__getOwnPropNames=Object.getOwnPropertyNames,__getProtoOf=Object.getPrototypeOf,__hasOwnProp=Object.prototype.hasOwnProperty,__export=(target,all)=>{for(var name in all)__defProp(target,name,{get:all[name],enumerable:!0})},__copyProps=(to,from,except,desc)=>{if(from&&"object"==typeof from\|\|"function"==typeof from)for(let key of __getOwnPropNames(from))__hasOwnProp.call(to,key)\|\|key===except\|\|__defProp(to,key,{get:()=>from[key],enumerable:!(desc=__getOwnPropDesc(from,key))\|\|desc.enumerable});return to},__toESM=(mod,isNodeMode,target)=>(target=null!=mod?__create(__getProtoOf(mod)):{},__copyProps(!isNodeMode&&mod&&mod.__esModule?target:__defProp(target,"default",{value:mod,enumerable:!0}),mod)),__toCommonJS=mod=>__copyProps(__defProp({},"__esModule",{value:!0}),mod),src_exports={};__export(src_exports,{performOneTask:()=>performOneTask,scraper:()=>scraper,setScraperLogFun:()=>setScraperLogFun,updateScraperConfig:()=>updateScraperConfig}),module.exports=__toCommonJS(src_exports);var import_controller=require("@letsscrapedata/controller"),import_proxy=require("@letsscrapedata/proxy"),import_utils=require("@letsscrapedata/utils"),pkgLog=import_utils.log;function setScraperLogFun(logFun){return"function"==typeof logFun&&(pkgLog=logFun,(0,import_controller.setControllerLogFun)(pkgLog),(0,import_proxy.setProxyLogFun)(pkgLog),!0)}async function logdbg(...args){await pkgLog(import_utils.LogLevel.DBG,...args)}async function loginfo(...args){await pkgLog(import_utils.LogLevel.INF,...args)}async function logwarn(...args){await pkgLog(import_utils.LogLevel.WRN,...args)}async function logerr(...args){await pkgLog(import_utils.LogLevel.ERR,...args)}(0,import_controller.setControllerLogFun)(pkgLog),(0,import_proxy.setProxyLogFun)(pkgLog),(0,import_utils.setLogLevel)(import_utils.LogLevel.DBG);var import_utils11=require("@letsscrapedata/utils"),import_xml2js_lsd=__toESM(require("xml2js-lsd"),1),defaultSplitOfParases="[\r\n\t,]+",defaultSeperatorOfParas="::",newLineChar="\n",defaultElementCfg={template:{tid:void 0,version:void 0,commited:!1},attrs:{},attr:{name:void 0,value:void 0},paras:{},para:{paraname:void 0,name:"",desc:"",uitype:"",defval:"",listid:"0",listparas:"",min:"",max:"",pattern:"",alert:""},element:{loc:void 0,source:"default",absolute:!1},iframe:{srcprefix:"",loc:""},templstr:{templ:void 0},elecontent_attr:{type:void 0,attrname:void 0,attrname2:"",attrname3:"",attrname4:"",trim:!1,pattern:"",boolattr:!1,multieles:!0,join:newLineChar,line:!1,absolute:!1},elecontent_imgbase64:{type:void 0},elecontent_innerhtml:{type:void 0},elecontent_length:{type:void 0},elecontent_outerhtml:{type:void 0},elecontent_text:{type:void 0,trim:!0,multieles:!0,join:newLineChar,line:!1,top:!1},decodefontsvg:{outtype:"text"},decodefontttf:{fflocs:"",intype:"outerhtml",outtype:"text"},ocrcfg_api:{type:void 0,nlchars:"",seperator:!1,lang:"zh"},ocrcfg_tesseract:{type:void 0,nlchars:"",seperator:!1,lang:"zh"},actions:{},action_api:{type:void 0,method:"GET",url:void 0,encodeuri:!1,session:!1,headerssource:"headers",headers:"",referer:"",proxy:!0,timeout:"30000",context:"default",data:"",datatype:"object",form:!1,path:"",diskcache:!0,cache:!0,varname:"",errname:"normal",valerrname:"ignore",pattern:"",flags:"",id:"0"},action_break:{type:void 0,id:"0"},action_captcha:{type:void 0,try:"1",errname:"captchafailed",debug:!1,id:"0"},action_click:{type:void 0,wait:"-1",gen:!0,cap:!0,clicktype:"click",try:"1",errname:"cfginvalid",popuppage:"ignored",popupsubtask:!1,login:!0,captcha:!0,navigate:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",downloadpath:"",filename:"",pathvarname:"",id:"0"},action_continue:{type:void 0,id:"0"},action_misc:{type:void 0,id:"0"},misc_delsyncdbdata:{type:void 0,tids:"",retention:"2592000",minretention:"86400"},misc_extractdata:{type:void 0,name:void 0,method:"put",paras:!0,execdata:!0,subtasks:!0,credits:!0},misc_getstatedata:{type:void 0,cookies:"new",localstorage:"new",headers:"none",userdata:"none"},action_exit:{type:void 0,errname:void 0,id:"0"},action_extract:{type:void 0,tabname:"dat_0000000000009911",id:"0"},action_extract_array:{type:void 0,subtype:void 0,list:void 0,requrl:"",subkeys:"",keys:"",tabname:"dat_0000000000009921",varname:"",idxname:"",maxloops:"0",errname:"cfginvalid",id:"0"},action_extract_script:{type:void 0,subtype:void 0,desc:"",base64:!1,html:!1,isolated:!0,tabname:"dat_0000000000009931",maxloops:"0",errname:"cfginvalid",id:"0"},action_extract_table:{type:void 0,subtype:void 0,orientation:"horizontal",check:!0,tabname:"dat_0000000000009941",keyformat:"camel",colnames:"",maxloops:"0",errname:"cfginvalid",id:"0"},column_element:{type:void 0,colname:void 0,completed:!1,setvar:!1,datatype:"string",errname:"ignore",valerrname:"ignore",pattern:"",flags:"",id:"0",discarded:!1},column_property:{type:void 0,subkeys:void 0,colname:void 0,completed:!1,setvar:!1,datatype:"string",valerrname:"ignore",pattern:"",flags:"",id:"0",discarded:!1},column_ocr:{type:void 0,colname:void 0,completed:!1,setvar:!1,datatype:"string",valerrname:"ignore",pattern:"",flags:"",id:"0",discarded:!1},column_subtask:{type:void 0,colname:void 0,completed:!1,setvar:!1,datatype:"string",valerrname:"ignore",pattern:"",flags:"",id:"0",discarded:!1},column_templstr:{type:void 0,colname:void 0,completed:!1,setvar:!1,datatype:"string",valerrname:"ignore",pattern:"",flags:"",id:"0",discarded:!1},action_goto:{type:void 0,url:void 0,reuse:!1,wait:"-1",encodeuri:!1,gen:!0,cap:!0,popuppage:"ignored",popupsubtask:!1,login:!0,captcha:!0,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",headers:"",referer:"",id:"0"},action_hover:{type:void 0,try:"1",gen:!1,cap:!1,wait:"-1",errname:"cfginvalid",id:"0"},action_ifelse:{type:void 0,id:"0"},condition_element:{type:void 0},condition_else:{type:void 0},condition_templstr:{type:void 0},action_input:{type:void 0,content:void 0,enter:!1,replace:!1,gen:!1,cap:!1,try:"1",wait:"-1",errname:"cfginvalid",popuppage:"ignored",popupsubtask:!1,login:!1,captcha:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",id:"0"},action_intercept_clear:{type:void 0,subtype:void 0},action_intercept_set:{type:void 0,subtype:void 0},request_abort:{type:void 0,method:"",url:"",resourcetype:"image",postdata:""},response_cache:{type:void 0,method:"",url:void 0,resourcetype:"",postdata:"",minsize:"0",maxsize:"0",contenttype:"text",responses:!0,tabname:"",requestheaders:!1,responseheaders:!1},request_data:{type:void 0,subkeys:void 0,value:void 0,datatype:void 0},request_header:{type:void 0,name:void 0,value:void 0},response_save:{type:void 0,method:"",url:void 0,resourcetype:"image",postdata:"",minsize:"0",maxsize:"0",basedir:"response",pathtype:"path_name",hashmethod:"MD5",hostnameparts:"0",extname:"",validextnames:"",params:"",encoding:"",tabname:""},response_statuscode:{type:void 0,codes:void 0,errname:void 0},action_loopdowhile_element:{type:void 0,subtype:void 0,iswhile:!1,varname:"",maxloops:"0",click:!1,navigate:!0,gen:!0,cap:!0,errname:"cfginvalid",wait:"-1",id:"0"},action_loopdowhile_templstr:{type:void 0,subtype:void 0,iswhile:!1,varname:"",maxloops:"0",id:"0"},action_loopfor:{type:void 0,from:void 0,to:void 0,step:"1",roundtype:"ceil",varname:void 0,maxloops:"0",errname:"ignore",id:"0"},action_loopineles:{type:void 0,varname:"",maxloops:"0",start:"1",end:"0",step:"1",errname:"ignore",id:"0"},action_loopinstr:{type:void 0,list:void 0,split:defaultSplitOfParases,varname:void 0,maxloops:"0",trim:!0,errname:"ignore",id:"0"},action_scroll_by:{type:void 0,subtype:void 0,height:"100",unit:"windowpct",maxtimes:"1",interval:"800",gen:!1,cap:!1,popuppage:"ignored",popupsubtask:!1,login:!1,captcha:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",id:"0"},action_scroll_intoview:{type:void 0,subtype:void 0,gen:!1,cap:!1,errname:"cfginvalid",popuppage:"ignored",popupsubtask:!1,login:!1,captcha:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",id:"0"},action_scroll_to:{type:void 0,subtype:void 0,height:"100",unit:"bodypct",gen:!1,cap:!1,popuppage:"ignored",popupsubtask:!1,login:!1,captcha:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",id:"0"},action_select:{type:void 0,selecttype:"value",selectval:"",gen:!1,cap:!1,try:"1",wait:"-1",errname:"cfginvalid",popuppage:"ignored",popupsubtask:!1,login:!1,captcha:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",id:"0"},dbquery:{dbname:void 0,sql:void 0,sqlparas:"",parasplit:defaultSeperatorOfParas,join:newLineChar},action_setvar_dbquery:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",errname:"ignore",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},action_setvar_element:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",try:"1",errname:"ignore",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},file:{path:void 0,encoding:"utf8"},action_setvar_file:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",proxy:!0,errname:"ignore",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},action_setvar_get:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",errname:"ignore",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},action_setvar_ocr:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",errname:"ignore",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},action_setvar_subtask:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",errname:"ignore",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},action_setvar_templstr:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",errname:"ignore",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},subtask:{tid:void 0,parasstr:void 0,idx:"",minlen:"1",errname:"cfginvalid",popuppage:!1},action_subtask:{type:void 0,subtasks:"",sameasparent:!1,id:"0"},action_wait_element:{type:void 0,subtype:void 0,timeout:"30000",state:"visible",errname:"ignore",wait:"-1",id:"0"},action_wait_navigation:{type:void 0,subtype:void 0,timeout:"30000",waituntil:"load",url:"",errname:"cfginvalid",wait:"-1",id:"0"},action_wait_sleep:{type:void 0,subtype:void 0,minms:"5000",maxms:"10000",errname:"ignore",wait:"-1",id:"0"},fonts:{},fontsvg:{exloc:"head link[rel='stylesheet']",inloc:"",csmaptype:"general",bsfilter:""},fontttf:{exloc:"head link[rel='stylesheet']",inloc:"",minuc:"57344",maxuc:"63744",startidx:"-1",fsfilter:"",fufilter:"",parsetype:""},fontselector:{name:void 0,fontfamily:void 0},fontfamily:{name:void 0,fontcodes:void 0,fontchars:void 0},fontcodes:{name:void 0,codes:void 0},fontchars:{name:void 0,chars:void 0},myfuns:{},myfun:{name:void 0,desc:"",base64:!1},captcha_amazon:{type:void 0},captcha_funcaptcha:{type:void 0},captcha_geetest:{type:void 0},captcha_keycaptcha:{type:void 0},captcha_mtcaptcha:{type:void 0},captcha_recaptcha:{type:void 0,minscore:"0.3"},captcha_turnstile:{type:void 0},image_element:{type:void 0},comment_element:{type:void 0,attr:"textContent"},input_element:{type:void 0,enter:!1},submit_element:{type:void 0},check_result:{type:void 0,attr:"",failedstr:""},captcha_text:{type:void 0,case:!1,lang:"en",math:!1,maxlen:"0",minlen:"0",question:!1,space:!1,texttype:"any"},captcha_coordinate:{type:void 0,lang:"en",maxclicks:"0",minclicks:"1"},transform:{defaultval:"",usevar:!1},fun_c_attr:{type:void 0,subtype:void 0,loc:void 0,idx:"0",attrname:void 0},fun_c_data:{type:void 0,subtype:void 0,loc:void 0,idx:"0",attrname:void 0},fun_c_html:{type:void 0,subtype:void 0,loc:void 0,idx:"0"},fun_c_length:{type:void 0,subtype:void 0,loc:void 0},fun_c_text:{type:void 0,subtype:void 0,loc:void 0,idx:"-9999"},fun_ceil:{type:void 0},fun_closingsubstr:{type:void 0,openchar:void 0,startstr:"",position:"0"},fun_compress:{type:void 0,method:"gzip",flush:"",finishflush:"",chunksize:"0",windowbits:"0",level:"",memlevel:"",strategy:"",sourceencoding:"utf8",targetencoding:"base64"},fun_concat:{type:void 0,str1:void 0,str2:"",str3:"",str4:"",str5:""},fun_convertencoding:{type:void 0,sourceencoding:"base64",targetencoding:"utf8",start:"0",end:"0"},fun_decompress:{type:void 0,method:"gunzip",flush:"",finishflush:"",chunksize:"0",windowbits:"0",sourceencoding:"base64",targetencoding:"utf8",start:"0",end:"0"},fun_decode:{type:void 0,str1:"",val1:"",str2:"",val2:"",str3:"",val3:"",str4:"",val4:"",str5:"",val5:"",defaultval:""},fun_floor:{type:void 0},fun_hashcode:{type:void 0,hashmethod:"MD5"},fun_includes:{type:void 0,searchstr:void 0},fun_insert:{type:void 0,str:void 0},fun_itemstoobj:{type:void 0,split:defaultSplitOfParases,kvsplit:":",keytrim:!0,keynows:!1,valuetrim:!0,valuenows:!1},fun_jsonparse:{type:void 0,key1:"",key2:"",key3:"",key4:"",key5:"",key6:"",keys:!1,length:!1,srctype:"json"},fun_length:{type:void 0},fun_matchall:{type:void 0,pattern:void 0,flags:"g",itemvalue:"match[0]",filter:"",start:"0",end:"0",resulttype:"all",join:newLineChar},fun_max:{type:void 0,split:defaultSplitOfParases},fun_min:{type:void 0,split:defaultSplitOfParases},fun_numeq:{type:void 0,num:void 0},fun_numge:{type:void 0,num:void 0},fun_numgt:{type:void 0,num:void 0},fun_numle:{type:void 0,num:void 0},fun_numlt:{type:void 0,num:void 0},fun_notincludes:{type:void 0,searchstr:void 0},fun_padend:{type:void 0,targetlen:void 0,padstr:" "},fun_padstart:{type:void 0,targetlen:void 0,padstr:" "},fun_parsefloat:{type:void 0},fun_parseint:{type:void 0},fun_randomarryidxes:{type:void 0,join:newLineChar},fun_replace:{type:void 0,substr:void 0,newstr:void 0,regexp:!1,flags:""},fun_round:{type:void 0},fun_setcomplement:{type:void 0,array2:void 0,split1:defaultSplitOfParases,split2:defaultSplitOfParases,sorttype:"nosort",valtype:"number",limit:"0",join:newLineChar},fun_setdifference:{type:void 0,array2:void 0,split1:defaultSplitOfParases,split2:defaultSplitOfParases,sorttype:"nosort",valtype:"number",limit:"0",join:newLineChar},fun_setintersection:{type:void 0,array2:void 0,split1:defaultSplitOfParases,split2:defaultSplitOfParases,sorttype:"nosort",valtype:"number",limit:"0",join:newLineChar},fun_setunion:{type:void 0,array2:void 0,split1:defaultSplitOfParases,split2:defaultSplitOfParases,sorttype:"nosort",valtype:"number",limit:"0",join:newLineChar},fun_slice:{type:void 0,beginidx:void 0,endidx:""},fun_shuffle:{type:void 0,split:defaultSplitOfParases},fun_sort:{type:void 0,split:defaultSplitOfParases,valtype:"number",sorttype:"asc",join:newLineChar},fun_specialconvert:{type:void 0,method:void 0,para1:"",para2:"",para3:"",para4:"",para5:""},fun_split:{type:void 0,split:defaultSplitOfParases,limit:"0",filter:"",start:"0",end:"0",resulttype:"all",val:"",insertstr:"",join:newLineChar},fun_streq:{type:void 0,str:void 0},fun_substrafter:{type:void 0,substr:void 0,position:"0",last:!1,include:!1},fun_substrbefore:{type:void 0,substr:void 0,position:"0",last:!1,include:!1},fun_tobool:{type:void 0},fun_tolowercase:{type:void 0},fun_tonum:{type:void 0},fun_touppercase:{type:void 0},fun_trim:{type:void 0},fun_trimend:{type:void 0},fun_trimstart:{type:void 0},fun_uniq:{type:void 0,split:defaultSplitOfParases,join:newLineChar},fun_urldelparams:{type:void 0,param1:void 0,param2:"",param3:""},fun_urldecode:{type:void 0,method:"URI",encoding:"utf8",percent:!0},fun_urlencode:{type:void 0,method:"URI",encoding:"utf8",percent:!0},fun_urlget:{type:void 0,attrorpath:void 0,hashmethod:"MD5",hostnameparts:"0",params:"",param:"",extname:"",validextnames:"",title:""},fun_urlsetparams:{type:void 0,param1:void 0,val1:"",param2:"",val2:"",param3:"",val3:"",replace:!0},fun_myfun:{type:void 0,name:void 0,arg1:"",arg2:"",arg3:"",arg4:"",arg5:""},get_addedtaskparas:{type:void 0,tid:void 0,limit:"10000",join:newLineChar,interval:"3600"},get_content:{type:void 0},get_cookies:{type:void 0,urls:"",domain:"",name:"",path:"",value:!1},get_datetime:{type:void 0,format:"yyyy-MM-ddThh:mm:ss"},get_execedtaskparas:{type:void 0,tid:void 0,limit:"10000",join:newLineChar,interval:"3600"},get_file:{type:void 0,url:void 0,path:"",basedir:"files",pathtype:"path_name",hashmethod:"MD5",hostnameparts:"0",params:"",extname:"",validextnames:"",pathvarname:"",proxy:!0,headers:"",referer:"",setvar:!1},httpbrowser:{name:void 0,minversion:"0",maxversion:"0",httpversion:"0"},httpdevice:{device:void 0},httpheader:{name:void 0,source:"value",value:void 0,delete:!1},httpos:{os:void 0},get_httpheaders:{type:void 0,headermethod:"genLatestWindowsChrome",origheaders:"",browserlist:"",httpversion:"0"},get_mhtml:{type:void 0,path:"",basedir:"mhtml",pathtype:"title",hashmethod:"MD5",hostnameparts:"0",pathvarname:""},get_pdf:{type:void 0,path:"",basedir:"pdf",pathtype:"title",hashmethod:"MD5",hostnameparts:"0",scale:"1",displayheaderfooter:!1,headertemplate:"",footertemplate:"",printbackground:!1,landscape:!1,pageranges:"",format:"Letter",width:"",height:"",top:"",right:"",bottom:"",left:"",screen:!1,onepage:!1,hmargin:"",setvar:!1,pathvarname:""},get_queuedcntwithparas:{type:void 0,tid:void 0,parasstr:void 0,operator:"eq"},get_queuedtaskparas:{type:void 0,tid:void 0,limit:"10000",join:newLineChar,interval:"3600"},get_random:{type:void 0,min:void 0,max:void 0},get_response:{type:void 0,requrl:"",idx:"-1",length:!1},get_screenshot:{type:void 0,basedir:"screenshot",path:"",extname:"png",quality:"-1",fullpage:!1,x:"0",y:"0",width:"0",height:"0",omitbackground:!1,setvar:!1,pathvarname:""},get_special:{type:void 0,method:void 0,parasstr:""},get_title:{type:void 0,aspath:!1},get_window:{type:void 0,key1:void 0,key2:"",key3:"",key4:"",key5:"",key6:"",keys:!1,length:!1}},import_controller2=require("@letsscrapedata/controller"),appConfig={XML_ATTR_KEY:"$$",XML_CHAR_KEY:"_",XML_CHILDREN_KEY:"children",GEN_CREDITS_BITS:16,GEN_CREDITS_AND:65535,CHECK_DNS_DOMAIN:"example.com",CHECK_POPUP_PAGE_TIMEOUT1:35e3,CHECK_POPUP_PAGE_TIMEOUT2:5e3,RESERVED_PARA_NAMES:["authInfo","inParas","sysParas"],EVENT_TYPES:{lsdPageClose:import_controller2.LsdPageEvent.PAGE_CLOSE,lsdPagePopup:import_controller2.LsdPageEvent.PAGE_POPUP,lsdScraperPopup:"lsdScraperPopup",taskComplete:"taskComplete"},OPERATE_ACTIONS:["click","goto","hover","input","scroll","select","wait","captcha"],performOneTask:console.log,cfg:{exitWhenCompleted:!1,useParasstrInXmlIfNeeded:!1,loadUnfinishedTasks:!1,loadFailedTasksInterval:0,baseDir:"",baseDownloadDir:"",baseFileDir:"",templateDir:"",taskDir:"",exportsDir:"",inputFileDirPart:"LetsScrapeData",useStealthPlugin:!1,browserControllerType:"patchright",browserType:"chromium",lsdLaunchOptions:{headless:!1,geoip:!0},lsdConnectOptions:{browserUrl:""},browserConfigs:[],captcha:{clientKey:"",captchas:[]},maxConcurrency:1,readCode:"",templateParas:[],urlPrefix:"https://api.letsscrapedata.com/nologin",totalMaxConcurrency:10,minMiliseconds:2e3,moveDataWhenStart:!1,dataFileFormat:"jsonl",columnSeperator:"::"}},TaskParser=class _TaskParser{static#applyDefaultElementCfg(cfg,parentEleName){try{if(!cfg\|\|!parentEleName)throw new Error(`##te Invalid paras, parentEleName ${parentEleName} in _applyDefaultElementCfg`);if("object"!=typeof cfg)return!0;const aKey=appConfig.XML_ATTR_KEY,cKey=appConfig.XML_CHILDREN_KEY;cfg[aKey]\|\|(cfg[aKey]={});let defaultCfgName=parentEleName;cfg[aKey].type&&(defaultCfgName=defaultCfgName+"_"+cfg[aKey].type),cfg[aKey].subtype&&(defaultCfgName=defaultCfgName+"_"+cfg[aKey].subtype);const defaultEleCfg=defaultElementCfg[defaultCfgName];if(!defaultEleCfg)throw new Error(`##te No default Cfg for eleName ${defaultCfgName} in _applyDefaultElementCfg`);const eleJsonCfg=Object.assign({},defaultEleCfg);if(Object.keys(defaultEleCfg).forEach((attrName=>{let attrVal=cfg[aKey][attrName];if("boolean"==typeof eleJsonCfg[attrName])"true"===attrVal\|\|""===attrVal?eleJsonCfg[attrName]=!0:"false"===attrVal&&(eleJsonCfg[attrName]=!1);else if(attrVal\|\|""===attrVal)eleJsonCfg[attrName]=attrVal;else if(void 0===eleJsonCfg[attrName])throw new Error(`##te Attribute ${attrName} of Element ${defaultCfgName} is undefined`)})),cfg[aKey]=eleJsonCfg,!cfg[cKey])return!0;const childKeys=Array.from(Object.keys(cfg[cKey]));for(const childEleName of childKeys){const childs=cfg[cKey][childEleName];1===childs.length&&""===childs[0]?cfg[cKey][childEleName]=[]:childs.forEach((cfg2=>_TaskParser.#applyDefaultElementCfg(cfg2,childEleName)))}return!0}catch(err){throw logerr(`##te Error parentEleName ${parentEleName} in _applyDefaultElementCfg`),new Error(err)}}static async convertXmlToJson(xmlStr,defaultCfgFlag=!0){try{const getTagName=function(name){const idx=name.indexOf("_");return idx>0?name.slice(0,idx):name},options={attrkey:appConfig.XML_ATTR_KEY,charkey:appConfig.XML_CHAR_KEY,childkey:appConfig.XML_CHILDREN_KEY,explicitChildren:!0,tagNameProcessors:[getTagName]},cKey=appConfig.XML_CHILDREN_KEY,root=await import_xml2js_lsd.default.parseStringPromise(xmlStr,options);if(!(root&&root.template&&root.template[cKey]&&Array.isArray(root.template[cKey].actions)))throw new Error(`##te Invalid XML config string: ${xmlStr}`);return defaultCfgFlag&&_TaskParser.#applyDefaultElementCfg(root.template,"template"),root.template}catch(err){throw logerr("##te Exeception in convertXmlToJson"),logerr(`##te xmlCfgStr: ${xmlStr}`),new Error(err)}}static getPartOfJsonCfg(jsonCfg,partName,optional=!1){try{const ret=jsonCfg[appConfig.XML_CHILDREN_KEY][partName][0];return ret\|\|null}catch(err){if(optional)return null;throw logerr(`##te Exeception when try to get ${partName} in getPartOfJsonCfg`),new Error(err)}}static#getAttrCfgsFromJsonCfg(jsonCfg){try{const attrs=_TaskParser.getPartOfJsonCfg(jsonCfg,"attrs");return!Array.isArray(attrs?.children?.attr)\|\|attrs.children.attr.length<=0?[]:attrs.children.attr.map((a=>a[appConfig.XML_ATTR_KEY]))}catch(err){return logerr(err),[]}}static getParaCfgsFromJsonCfg(jsonCfg){try{const paras=_TaskParser.getPartOfJsonCfg(jsonCfg,"paras");return!Array.isArray(paras?.children?.para)\|\|paras.children.para.length<=0?[]:paras.children.para.map((p=>p[appConfig.XML_ATTR_KEY]))}catch(err){return logerr(err),[]}}static getMyfunCfgsFromJsonCfg(jsonCfg){try{const myfuns=_TaskParser.getPartOfJsonCfg(jsonCfg,"myfuns",!0);return!Array.isArray(myfuns?.children?.myfun)\|\|myfuns.children.myfun.length<=0?[]:myfuns.children.myfun.map((p=>Object.assign({},p[appConfig.XML_ATTR_KEY],{script:p[appConfig.XML_CHAR_KEY]})))}catch(err){return logerr(err),[]}}static getAttrsInXml(jsonCfg){const attrConfigs=_TaskParser.#getAttrCfgsFromJsonCfg(jsonCfg),attrsInXml={};return attrConfigs.forEach((attrConfig=>attrsInXml[attrConfig.name]=attrConfig.value)),attrsInXml}static#getAllActionsOfType(actions,actionType){const extracts=[],cKey=appConfig.XML_CHILDREN_KEY;for(const action of actions)if(action?.[appConfig.XML_ATTR_KEY]?.type===actionType)extracts.push(action);else if(action&&action[cKey]&&(action[cKey].action\|\|action[cKey].condition)){_TaskParser.#getAllActionsOfType(action[cKey].action?action[cKey].action:action[cKey].condition,actionType).forEach((act=>{extracts.push(act)}))}return extracts}static getCaptchTypes(jsonCfg){const cKey=appConfig.XML_CHILDREN_KEY,rootActions=jsonCfg[cKey].actions[0][cKey].action,captchaCfgs=_TaskParser.#getAllActionsOfType(rootActions,"captcha"),captchaTypes=[];for(const captchaCfg of captchaCfgs){const captchas=captchaCfg[appConfig.XML_CHILDREN_KEY]?.captcha;Array.isArray(captchas)&&captchas.forEach((cfg=>{const captchaType=cfg[appConfig.XML_ATTR_KEY].type;captchaType&&captchaTypes.push(captchaType)}))}return captchaTypes}static getInParas(parasStr,paraCfgs=[],splitStr="::"){const inParas={},parts=parasStr.split("@@"),prefixes=["para","apara","rpara"];for(let idx=0;idx<3;idx++){let prefix=prefixes[idx],paraNum=paraCfgs.filter((p=>p.paraname.startsWith(prefix))).length;if(parts.length>idx&&parts[idx].length>0){let paras=parts.length>idx&&parts[idx].length>0?parts[idx].split(splitStr):[];for(let i=1;i<=paras.length;i++)inParas[`${prefix}${i}`]=paras[i-1];for(let i=paras.length+1;i<=paraNum;i++)inParas[`${prefix}${i}`]=""}}return inParas}},fs=__toESM(require("fs"),1),path=__toESM(require("path"),1),import_utils2=require("@letsscrapedata/utils"),TemplateManagerInScraper=class _TemplateManagerInScraper{static#templateMap=new Map;static#failedTempateIdSet=new Set;static#getReadCode(templateId){const cfg=appConfig.cfg.templateParas.find((tmc=>tmc.templateId===templateId));return cfg?.readCode?cfg.readCode:appConfig.cfg.readCode}static async#getTemplateFromLocal(templateId,xmlStr){try{if(!xmlStr){if(!appConfig.cfg.templateDir)return null;const filenames=await(0,import_utils2.filesInDir)(appConfig.cfg.templateDir,"general",`tid${templateId}_`,"xml");let filename=1===filenames.length?filenames[0]:`tid${templateId}.xml`;filename=path.join(appConfig.cfg.templateDir,filename),xmlStr=fs.readFileSync(filename,"utf8")}if(!xmlStr)return null;return{templateId:templateId,domainId:1,defaultElementSource:"browser",maxExecutionDuration:600,configDetail:xmlStr}}catch(err){return null}}static async#getTemplateFromServer(templateId,code){try{if(_TemplateManagerInScraper.#failedTempateIdSet.has(templateId))return null;let url=`${appConfig.cfg.urlPrefix}/template`;url=`${url}?tid=${templateId}&code=${code}`;const response=await fetch(url);if(200!==response.status)return 403===response.status?logwarn(`Failed to get template ${templateId}: No privilege`):logwarn(`Failed to get template ${templateId}, status: ${response.status}`),_TemplateManagerInScraper.#failedTempateIdSet.add(templateId),null;const json=await response.json();if(!json\|\|0!==json.code\|\|!json.data)return logwarn(`Failed to get template ${templateId}: ${json.message}`),null;if(json.data.templateId!==templateId)return logerr(`Different templateId ${templateId} / ${json.templateId}`),null;const{domainId:domainId=0,defaultElementSource:defaultElementSource,configDetail:configDetail="",maxExecutionDuration:maxExecutionDuration}=json.data;if(!configDetail)return logerr(`Invalid content in template ${templateId}`),null;return{templateId:templateId,domainId:domainId,defaultElementSource:defaultElementSource,maxExecutionDuration:maxExecutionDuration,configDetail:configDetail}}catch(err){return logdbg(err),null}}static async parseXmlTemplate(xmlStr){const jsonCfg=await TaskParser.convertXmlToJson(xmlStr),paraCfgs=TaskParser.getParaCfgsFromJsonCfg(jsonCfg),fontsConfig=TaskParser.getPartOfJsonCfg(jsonCfg,"fonts",!0),myfunCfgs=TaskParser.getMyfunCfgsFromJsonCfg(jsonCfg),actionConfigs=TaskParser.getPartOfJsonCfg(jsonCfg,"actions",!1),attrsInXml=TaskParser.getAttrsInXml(jsonCfg),captchaTypes=TaskParser.getCaptchTypes(jsonCfg),currentTime=(0,import_utils2.getCurrentUnixTime)(),myfunScripts={};myfunCfgs.forEach((myfunCfg=>{const{name:name,script:script,base64:base64}=myfunCfg;if(base64){const str=(0,import_utils2.decodeFromBase64)(script);"string"==typeof str&&str&&(myfunScripts[name]=str)}else myfunScripts[name]=script}));return{actionConfigs:actionConfigs,paraCfgs:paraCfgs,fontsConfig:fontsConfig\|\|null,myfunScripts:myfunScripts,attrsInXml:attrsInXml,captchaTypes:captchaTypes,lastUsedTime:currentTime,lastCheckTime:currentTime}}static async getTemplateConfig(templateId,xmlStr=""){if(!xmlStr){const cachedParsedTemplateExt=_TemplateManagerInScraper.#templateMap.get(templateId);if(cachedParsedTemplateExt)return cachedParsedTemplateExt}let template=await _TemplateManagerInScraper.#getTemplateFromLocal(templateId,xmlStr);if(!template){const code=_TemplateManagerInScraper.#getReadCode(templateId);template=await _TemplateManagerInScraper.#getTemplateFromServer(templateId,code)}if(!template){const message=`Cannot get template tid ${templateId}`;throw logwarn(message),new Error(message)}const parsedTemplate=await _TemplateManagerInScraper.parseXmlTemplate(template.configDetail),parsedTemplateExt=Object.assign({},parsedTemplate,{template:template});return _TemplateManagerInScraper.#templateMap.set(templateId,parsedTemplateExt),parsedTemplateExt}static clearTemplateConfig(templateId=0){return templateId>0?_TemplateManagerInScraper.#templateMap.delete(templateId):_TemplateManagerInScraper.#templateMap.clear(),!0}},import_node_events=__toESM(require("events"),1),import_node_events2=require("events"),fs6=__toESM(require("fs"),1),path5=__toESM(require("path"),1),import_got_scraping=require("got-scraping"),import_isolated_vm2=__toESM(require("isolated-vm"),1),import_utils9=require("@letsscrapedata/utils"),import_controller3=require("@letsscrapedata/controller"),uuid=__toESM(require("uuid"),1),SysParas=class{#taskContext;#templateId;#taskId;#version;constructor(taskContext,templateId,taskId,version){this.#taskContext=taskContext,this.#templateId=templateId,this.#taskId=taskId,this.#version=version}get version(){return this.#version}get templateId(){return this.#templateId}get taskId(){return this.#taskId}get url(){try{return this.#taskContext?.page?this.#taskContext.page.url():""}catch{return""}}get hasPopupPage(){return this.#taskContext?.popupPage?"1":"0"}get subtaskNum(){const subtasks=this.#taskContext?.subtasks;return Array.isArray(subtasks)?String(subtasks.length):"0"}get datatableNum(){const execData=this.#taskContext?.execData;return execData&&"object"==typeof execData?String(Object.keys(execData).length):"0"}get responses(){try{const responses=this.#taskContext?.responses;return Array.isArray(responses)?JSON.stringify(responses):""}catch(err){return""}}get currentDate(){return(new Date).toISOString().slice(0,10)}get currentDateTime(){return(new Date).toISOString().slice(0,19)}get currentIsoTime(){return(new Date).toISOString()}get currentLocalTime(){const date=new Date;return`${date.getFullYear()}-${String(date.getMonth()+1).padStart(2,"0")}-${String(date.getDate()).padStart(2,"0")} ${String(date.getHours()).padStart(2,"0")}:${String(date.getMinutes()).padStart(2,"0")}:${String(date.getSeconds()).padStart(2,"0")}`}get currentUtcTime(){return(new Date).toISOString().slice(0,19).replace("T"," ")}get currentTime(){return(new Date).toISOString().slice(11,19)}get currentUnixMs(){return String(Date.now())}get currentUnixSec(){return String(Math.floor(Date.now()/1e3))}get random10(){return String(Math.floor(10Math.random()))}get random100(){return String(Math.floor(100Math.random()))}get random1000(){return String(Math.floor(1e3Math.random()))}get random10000(){return String(Math.floor(1e4Math.random()))}get uuidv1(){return uuid.v1()}get uuidv4(){return uuid.v4()}},iconv=__toESM(require("iconv-lite"),1),zlib=__toESM(require("zlib"),1),cheerio=__toESM(require("cheerio"),1),import_isolated_vm=__toESM(require("isolated-vm"),1),import_utils3=require("@letsscrapedata/utils"),LsdString=class _LsdString{static specialConvertFunObj={};static setSpecialConvertFunObj(specialConvertFunObj){return _LsdString.specialConvertFunObj=specialConvertFunObj,!0}static transformStrSync(origStr,funCfgs,defaultVal){let str=origStr;try{if("string"!=typeof origStr\|\|!Array.isArray(funCfgs))throw new Error("Invalid paras in transformStr");for(const funCfg of funCfgs){if("object"!=typeof funCfg)throw new Error("Invalid funCfg in transformStr");const functionName="c"===funCfg.type?`${funCfg.type}_${funCfg.subtype}`:funCfg.type;let fun=_LsdString._funPerformers[functionName];fun&&"function"==typeof fun?str=fun(str,funCfg):logerr(`Invalid funName ${funCfg.type} in transformStr`)}}catch(err){loginfo(err)}return str\|\|defaultVal}static async transformStr(origStr,funCfgs,defaultVal,myfunScripts){let str=origStr;try{if("string"!=typeof origStr\|\|!Array.isArray(funCfgs))throw new Error("Invalid paras in transformStr");for(const funCfg of funCfgs){if("object"!=typeof funCfg)throw new Error("Invalid funCfg in transformStr");const functionName="c"===funCfg.type?`${funCfg.type}_${funCfg.subtype}`:funCfg.type;if("myfun"!==functionName){let fun=_LsdString._funPerformers[functionName];if(!fun\|\|"function"!=typeof fun){logerr(`Invalid funName ${funCfg.type} in transformStr`);continue}str=fun(str,funCfg)}else str=await _LsdString.#myfun(str,funCfg,myfunScripts)}}catch(err){loginfo(err)}return str\|\|defaultVal}static#getStdFunScriptFun=null;static setGetStdFunScript(fun){return _LsdString.#getStdFunScriptFun=fun,!0}static async#getScriptOfMyfun(name,myfunScripts){if("string"==typeof name&&name){if(myfunScripts?.[name])return myfunScripts[name];if(name.startsWith("std")&&_LsdString.#getStdFunScriptFun){const script=await _LsdString.#getStdFunScriptFun(name);if(script)return script}}return""}static async#myfun(str,funCfg,myFunScripts){const{name:name,arg1:arg1,arg2:arg2,arg3:arg3,arg4:arg4,arg5:arg5}=funCfg;try{const script=await _LsdString.#getScriptOfMyfun(name,myFunScripts);if(!script)return"";const inData={origStr:str,arg1:arg1,arg2:arg2,arg3:arg3,arg4:arg4,arg5:arg5},isolate=new import_isolated_vm.default.Isolate,context=isolate.createContextSync(),externalCopy=new import_isolated_vm.default.ExternalCopy(inData);context.global.setSync("inData",externalCopy.copyInto());const content=isolate.compileScriptSync(script).runSync(context);return"string"!=typeof content&&logwarn("Please check the script of myfun is correct."),"string"==typeof content?content:""}catch(err){return logwarn(err.message),""}}static __getElementBySelector($,loc,idx=0){try{const eles="root"===loc?$.root():$(loc);return eles.length<=0\|\|-9999===idx?eles:eles.eq(idx)}catch(err){return logerr("##te cheerio invalid $ or loc"),null}}static __cheerioOptions={xmlMode:!0,decodeEntities:!1};static _c_attr(origStr,funCfg){try{const{loc:loc,idx:idx,attrname:attrname}=funCfg,idxInt=parseInt(idx),$=cheerio.load(origStr,_LsdString.__cheerioOptions);return _LsdString.__getElementBySelector($,loc,idxInt).attr(attrname)}catch(err){return logerr(`##tmp cheerio c_attr err: ${err}`),""}}static _c_data(origStr,funCfg){try{const{loc:loc,idx:idx,attrname:attrname}=funCfg,idxInt=parseInt(idx),$=cheerio.load(origStr,_LsdString.__cheerioOptions);return _LsdString.__getElementBySelector($,loc,idxInt).data(attrname)}catch(err){return logerr(`##tmp cheerio c_data err: ${err}`),""}}static _c_html(origStr,funCfg){try{const{loc:loc,idx:idx}=funCfg,idxInt=parseInt(idx),$=cheerio.load(origStr,_LsdString.__cheerioOptions);return _LsdString.__getElementBySelector($,loc,idxInt).html()}catch(err){return logerr(`##tmp cheerio c_html err: ${err}`),""}}static _c_length(origStr,funCfg){try{const{loc:loc}=funCfg,idxInt=-9999,$=cheerio.load(origStr,_LsdString.__cheerioOptions),$$=_LsdString.__getElementBySelector($,loc,idxInt);return String($$.length)}catch(err){return logerr(`##tmp cheerio c_length err: ${err}`),"0"}}static _c_text(origStr,funCfg){try{const{loc:loc,idx:idx}=funCfg,idxInt=parseInt(idx),$=cheerio.load(origStr,_LsdString.__cheerioOptions);return _LsdString.__getElementBySelector($,loc,idxInt).text()}catch(err){return logerr(`##tmp cheerio c_text err: ${err}`),""}}static _ceil(origStr){return String(Math.ceil(Number(origStr)))}static _closingsubstr(origStr,funCfg){const{openchar:openchar,startstr:startstr,position:position}=funCfg;if("string"!=typeof origStr\|\|"string"!=typeof openchar\|\|!["{","[","(","lt"].includes(openchar)\|\|"string"!=typeof startstr\|\|"number"!=typeof parseInt(position)\|\|parseInt(position)<0)return"";const openChar="lt"===openchar?"<":openchar,closingChar={"{":"}","[":"]","(":")","<":">"}[openChar];let openBracketCount=1,startPos=-1;if(startstr){if(startPos=origStr.indexOf(startstr),startPos<0)return""}else startPos=parseInt(position);const startIdx=origStr.indexOf(openChar,startPos);if(startIdx<0)return-1;let idx=startIdx+1;for(;idx<origStr.length;idx++){let char=origStr[idx];if(char===openChar)openBracketCount++;else if(char===closingChar&&(openBracketCount--,0===openBracketCount))break}const endIdx=idx<=origStr.length?idx+1:-1;return endIdx>startIdx?origStr.substring(startIdx,endIdx):""}static __getCompressOptions(funCfg,compressFlag){return funCfg\|\|compressFlag?void 0:{}}static _compress(origStr,funCfg){try{const{method:method,sourceencoding:sourceencoding,targetencoding:targetencoding,start:start,end:end}=funCfg,options=_LsdString.__getCompressOptions(funCfg,!0),origBuffer=Buffer.from(origStr,sourceencoding);let compressedBuffer=null;return"gzip"===method?compressedBuffer=zlib.gzipSync(origBuffer,options):"deflate"===method?compressedBuffer=zlib.deflateSync(origBuffer,options):"deflateraw"===method?compressedBuffer=zlib.deflateRawSync(origBuffer,options):"brotli"===method&&(compressedBuffer=zlib.brotliCompressSync(origBuffer,options)),compressedBuffer?_LsdString.__convertBufferToString(compressedBuffer,targetencoding,start,end):""}catch(err){return""}}static _decompress(origStr,funCfg){try{const{method:method,sourceencoding:sourceencoding,targetencoding:targetencoding,start:start,end:end}=funCfg,options=_LsdString.__getCompressOptions(funCfg,!1),origBuffer=Buffer.from(origStr,sourceencoding);let decompressedBuffer=Buffer.from(origStr,sourceencoding);return"gunzip"===method?decompressedBuffer=zlib.gunzipSync(origBuffer,options):"inflate"===method?decompressedBuffer=zlib.inflateSync(origBuffer,options):"inflateraw"===method?decompressedBuffer=zlib.inflateRawSync(origBuffer,options):"unzip"===method?decompressedBuffer=zlib.unzipSync(origBuffer,options):"brotli"===method&&(decompressedBuffer=zlib.brotliDecompressSync(origBuffer,options)),decompressedBuffer?_LsdString.__convertBufferToString(decompressedBuffer,targetencoding,start,end):""}catch(err){return""}}static _concat(origStr,funCfg){let str=origStr;const{str1:str1,str2:str2,str3:str3,str4:str4,str5:str5}=funCfg;return[str1,str2,str3,str4,str5].forEach((item=>{"string"==typeof item&&(str=str.concat(item))})),str}static _specialconvert(origStr,funCfg){let str=origStr;try{const{method:method}=funCfg;return"function"==typeof _LsdString.specialConvertFunObj[method]&&(str=_LsdString.specialConvertFunObj[method](origStr,funCfg)),str}catch(err){return str}}static _decode(origStr,funCfg){const{str1:str1,val1:val1,str2:str2,val2:val2,str3:str3,val3:val3,str4:str4,val4:val4,str5:str5,val5:val5,defaultval:defaultval}=funCfg;try{return origStr===str1?val1:origStr===str2?val2:origStr===str3?val3:origStr===str4?val4:origStr===str5?val5:defaultval}catch(err){return""}}static __convertBufferToString(buffer,encoding,startStr,endStr){const start=startStr?Number(startStr):0,end=endStr?Number(endStr):0;return buffer instanceof Buffer?end>0?buffer.toString(encoding,start,end):start>0?buffer.toString(encoding,start):buffer.toString(encoding):""}static _convertencoding(origStr,funCfg){const{sourceencoding:sourceencoding,targetencoding:targetencoding,start:start,end:end}=funCfg;try{const buffer=Buffer.from(origStr,sourceencoding);return _LsdString.__convertBufferToString(buffer,targetencoding,start,end)}catch(err){return""}}static _floor(origStr){return String(Math.floor(Number(origStr)))}static _hashcode(origStr,funCfg){const{hashmethod:hashmethod}=funCfg;return import_utils3.LsdUrl.getHashCode(origStr,hashmethod)}static _includes(origStr,funCfg){const{searchstr:searchstr}=funCfg;if("string"!=typeof searchstr)throw new Error("Invalid searchstr in _contains");return String(Number(origStr.includes(searchstr)))}static _insert(origStr,funCfg){const{str:str}=funCfg;if("string"!=typeof str)throw new Error("Invalid str in _insert");return str+origStr}static _itemstoobj(origStr,funCfg){const{split:itemSplit,kvsplit:kvSplit,keytrim:keyTrimFlag,keynows:keyNowsFlag,valuetrim:valueTrimFlag,valuenows:valueNowsFlag}=funCfg;if(!itemSplit\|\|!kvSplit)return"{}";try{let retObj={};const items=origStr.split(new RegExp(itemSplit)),reKvSplit=new RegExp(kvSplit);for(const item of items){const kv=item.split(reKvSplit);if(2!==kv.length)continue;let keyStr=keyTrimFlag?kv[0].trim():kv[0];keyNowsFlag&&(keyStr=keyStr.split("").filter((i=>i.trim())).join(""));let valueStr=valueTrimFlag?kv[1].trim():kv[1];valueNowsFlag&&(valueStr=valueStr.split("").filter((i=>i.trim())).join("")),keyStr&&(retObj[keyStr]=valueStr)}return JSON.stringify(retObj)}catch(err){return"{}"}}static _jsonparse(origStr,funCfg){const{key1:key1,key2:key2,key3:key3,key4:key4,key5:key5,key6:key6,keys:keys,length:length,srctype:srctype}=funCfg;let retObj=null;try{if("script"===srctype)return"";if(retObj=JSON.parse(origStr),"object"!=typeof retObj\|\|!retObj)return""}catch(err){return""}return retObj=(0,import_utils3.getMemberOfObject)(retObj,key1,key2,key3,key4,key5,key6),null==retObj?"":"string"==typeof retObj?retObj:"number"==typeof retObj?String(retObj):(keys&&(retObj=Array.from(Object.keys(retObj))),length&&Array.isArray(retObj)?String(retObj.length):JSON.stringify(retObj))}static _length(origStr){return String(origStr.length)}static _matchall(origStr,funCfg){try{const{pattern:pattern,flags:flags,itemvalue:itemvalue,filter:filter,start:start,end:end,resulttype:resulttype,join:join9}=funCfg,re=new RegExp(pattern,flags),matches=origStr.matchAll(re);if("boolean"===resulttype)return matches?"1":"0";if(!matches)return"";let strArray=[];for(const match of matches)if("match[0]"===itemvalue)strArray.push(match[0]);else{let item=itemvalue;for(let i=0;i<match.length;i++)item=item.replaceAll(`match[${i}]`,match[i]);match.index>=0&&(item=item.replaceAll("match.index",String(match.index))),strArray.push(item)}if(filter){const reFilter=new RegExp(filter);strArray=strArray.filter((item=>item.match(reFilter)))}const startInt=parseInt(start),endInt=parseInt(end);return 0===startInt&&0===endInt\|\|(strArray=0===endInt?strArray.slice(startInt):strArray.slice(startInt,endInt)),"all"===resulttype?strArray.join(join9):"length"===resulttype?String(strArray.length):""}catch(err){return loginfo(err),""}}static _max(origStr,funCfg){try{const{split:split}=funCfg,nums=origStr.split(new RegExp(split)).filter((item=>item.trim())).map((item=>Number(item)));return nums.length>0?String(nums.sort(((a,b)=>a-b)).pop()):""}catch(err){return loginfo(err),""}}static _min(origStr,funCfg){try{const{split:split}=funCfg,nums=origStr.split(new RegExp(split)).filter((item=>item.trim())).map((item=>Number(item)));return nums.length>0?String(nums.sort(((a,b)=>a-b))[0]):""}catch(err){return loginfo(err),""}}static _numeq(origStr,funCfg){const{num:num}=funCfg;return"string"!=typeof origStr\|\|"string"!=typeof num?"0":origStr.trim().match(/^\d+$/)&&num.trim().match(/^\d+$/)&&Number(origStr)===Number(num)?"1":"0"}static _numge(origStr,funCfg){const{num:num}=funCfg;return"string"!=typeof origStr\|\|"string"!=typeof num?"0":origStr.trim().match(/^\d+$/)&&num.trim().match(/^\d+$/)&&Number(origStr)>=Number(num)?"1":"0"}static _numgt(origStr,funCfg){const{num:num}=funCfg;return"string"!=typeof origStr\|\|"string"!=typeof num?"0":origStr.trim().match(/^\d+$/)&&num.trim().match(/^\d+$/)&&Number(origStr)>Number(num)?"1":"0"}static _numle(origStr,funCfg){const{num:num}=funCfg;return"string"!=typeof origStr\|\|"string"!=typeof num?"0":origStr.trim().match(/^\d+$/)&&num.trim().match(/^\d+$/)&&Number(origStr)<=Number(num)?"1":"0"}static _numlt(origStr,funCfg){const{num:num}=funCfg;return"string"!=typeof origStr\|\|"string"!=typeof num?"0":origStr.trim().match(/^\d+$/)&&num.trim().match(/^\d+$/)&&Number(origStr)<Number(num)?"1":"0"}static _notincludes(origStr,funCfg){const{searchstr:searchstr}=funCfg;if("string"!=typeof searchstr)throw new Error("Invalid str in _notcontains");return String(Number(!origStr.includes(searchstr)))}static _padEnd(origStr,funCfg){const{targetlen:targetlen,padstr:padstr}=funCfg,targetLen=parseInt(targetlen),padStr="string"==typeof padstr&&padstr.length>0?padstr:" ";return origStr.padEnd(targetLen,padStr)}static _padStart(origStr,funCfg){const{targetlen:targetlen,padstr:padstr}=funCfg,targetLen=parseInt(targetlen),padStr="string"==typeof padstr&&padstr.length>0?padstr:" ";return origStr.padStart(targetLen,padStr)}static _parseFloat(origStr){return String(parseFloat(origStr))}static _parseInt(origStr){return String(parseInt(origStr))}static _randomarryidxes(origStr,funCfg){const{join:join9}=funCfg;try{const length=parseInt(origStr);return[...Array(length).keys()].sort((()=>Math.random()-.5)).join(join9)}catch(err){return""}}static _replace(origStr,funCfg){const{substr:substr,newstr:newstr,regexp:regexp,flags:flags}=funCfg;try{const regexpFlag=regexp;if("string"!=typeof substr)throw new Error("Invalid substr in _replace");if("string"!=typeof newstr)throw new Error("Invalid newstr in _replace");return regexpFlag?origStr.replace(RegExp(substr,flags),newstr):origStr.replace(substr,newstr)}catch(err){return origStr}}static _round(origStr){return String(Math.round(Number(origStr)))}static __sort(origArr,valtype,sorttype){let retArr;return retArr="string"===valtype?"asc"===sorttype?origArr.sort(((a,b)=>a.localeCompare(b))):origArr.sort(((a,b)=>b.localeCompare(a))):"asc"===sorttype?origArr.sort(((a,b)=>parseFloat(a)-parseFloat(b))):origArr.sort(((a,b)=>parseFloat(b)-parseFloat(a))),retArr}static _setcomplement(origStr,funCfg){const{array2:array2,split1:split1,split2:split2,sorttype:sorttype,valtype:valtype,limit:limit,join:join9}=funCfg;let reSplit1,reSplit2;try{reSplit1=new RegExp(split1),reSplit2=new RegExp(split2)}catch(err){throw new Error(`Invalid regexp split1: ${split1}, split2: ${split2} in _setcomplement`)}try{const limitInt=parseInt(limit);if(!["nosort","asc","desc"].includes(sorttype)\|\|!["string","number"].includes(valtype)\|\|limitInt<0\|\|"string"!=typeof array2\|\|"string"!=typeof join9)throw new Error(`Invalid limit: ${limit}, sorttype: ${sorttype}, or array2 in _setcomplement`);const strArr1=origStr.split(reSplit1),strArr2=array2.split(reSplit2);let resArr=(0,import_utils3.getComplementOfTwoArray)(strArr1,strArr2);return["asc","desc"].includes(sorttype)&&(resArr=_LsdString.__sort(resArr,valtype,sorttype)),limitInt>0?resArr.slice(0,limitInt).join(join9):resArr.join(join9)}catch(err){return loginfo(err),origStr}}static _setdifference(origStr,funCfg){const{array2:array2,split1:split1,split2:split2,sorttype:sorttype,valtype:valtype,limit:limit,join:join9}=funCfg;let reSplit1,reSplit2;try{reSplit1=new RegExp(split1),reSplit2=new RegExp(split2)}catch(err){throw new Error(`Invalid regexp split1: ${split1}, split2: ${split2} in _setcomplement`)}try{const limitInt=parseInt(limit);if(!["nosort","asc","desc"].includes(sorttype)\|\|!["string","number"].includes(valtype)\|\|limitInt<0\|\|"string"!=typeof array2\|\|"string"!=typeof join9)throw new Error(`Invalid limit: ${limit}, sorttype: ${sorttype}, or array2 in _setcomplement`);const strArr1=origStr.split(reSplit1),strArr2=array2.split(reSplit2);let resArr=(0,import_utils3.getDifferenceOfTwoArray)(strArr1,strArr2);return["asc","desc"].includes(sorttype)&&(resArr=_LsdString.__sort(resArr,valtype,sorttype)),limitInt>0?resArr.slice(0,limitInt).join(join9):resArr.join(join9)}catch(err){return loginfo(err),origStr}}static _setintersection(origStr,funCfg){const{array2:array2,split1:split1,split2:split2,sorttype:sorttype,valtype:valtype,limit:limit,join:join9}=funCfg;let reSplit1,reSplit2;try{reSplit1=new RegExp(split1),reSplit2=new RegExp(split2)}catch(err){throw new Error(`Invalid regexp split1: ${split1}, split2: ${split2} in _setcomplement`)}try{const limitInt=parseInt(limit);if(!["nosort","asc","desc"].includes(sorttype)\|\|!["string","number"].includes(valtype)\|\|limitInt<0\|\|"string"!=typeof array2\|\|"string"!=typeof join9)throw new Error(`Invalid limit: ${limit}, sorttype: ${sorttype}, or array2 in _setcomplement`);const strArr1=origStr.split(reSplit1),strArr2=array2.split(reSplit2);let resArr=(0,import_utils3.getIntersectionOfTwoArray)(strArr1,strArr2);return["asc","desc"].includes(sorttype)&&(resArr=_LsdString.__sort(resArr,valtype,sorttype)),limitInt>0?resArr.slice(0,limitInt).join(join9):resArr.join(join9)}catch(err){return loginfo(err),origStr}}static _setunion(origStr,funCfg){const{array2:array2,split1:split1,split2:split2,sorttype:sorttype,valtype:valtype,limit:limit,join:join9}=funCfg;let reSplit1,reSplit2;try{reSplit1=new RegExp(split1),reSplit2=new RegExp(split2)}catch(err){throw new Error(`Invalid regexp split1: ${split1}, split2: ${split2} in _setcomplement`)}try{const limitInt=parseInt(limit);if(!["nosort","asc","desc"].includes(sorttype)\|\|!["string","number"].includes(valtype)\|\|limitInt<0\|\|"string"!=typeof array2\|\|"string"!=typeof join9)throw new Error(`Invalid limit: ${limit}, sorttype: ${sorttype}, or array2 in _setcomplement`);const strArr1=origStr.split(reSplit1),strArr2=array2.split(reSplit2);let resArr=(0,import_utils3.getUnionOfTwoArray)(strArr1,strArr2);return["asc","desc"].includes(sorttype)&&(resArr=_LsdString.__sort(resArr,valtype,sorttype)),limitInt>0?resArr.slice(0,limitInt).join(join9):resArr.join(join9)}catch(err){return loginfo(err),origStr}}static _shuffle(origStr,funCfg){try{const{split:split}=funCfg;if("string"!=typeof split)throw new Error("Invalid split in _shuffle");const splitArr=origStr.split(new RegExp(split));return!Array.isArray(splitArr)\|\|splitArr.length<=1?origStr:splitArr.sort((()=>Math.random()-.5)).join(split)}catch(err){return loginfo(err),origStr}}static _slice(origStr,funCfg){const{beginidx:beginidx,endidx:endidx}=funCfg,beginIdx=parseInt(beginidx),endIdx="string"==typeof endidx&&endidx.length>0?parseInt(endidx):void 0;return origStr.slice(beginIdx,endIdx)}static _sort(origStr,funCfg){try{const{split:split,valtype:valtype,sorttype:sorttype,join:join9}=funCfg;if("string"!=typeof split\|\|!["string","number"].includes(valtype)\|\|!["asc","desc"].includes(sorttype))throw new Error("Invalid split or valtype in _sort");const splitArr=origStr.split(new RegExp(split));return!Array.isArray(splitArr)\|\|splitArr.length<=1?origStr:"string"===valtype?"asc"===sorttype?splitArr.sort(((a,b)=>a.localeCompare(b))).join(join9):splitArr.sort(((a,b)=>b.localeCompare(a))).join(join9):"asc"===sorttype?splitArr.sort(((a,b)=>parseFloat(a)-parseFloat(b))).join(join9):splitArr.sort(((a,b)=>parseFloat(b)-parseFloat(a))).join(join9)}catch(err){return loginfo(err),""}}static _split(origStr,funCfg){try{const{split:split,limit:limit,filter:filter,start:start,end:end,resulttype:resulttype,val:val,insertstr:insertstr,join:join9}=funCfg;if("string"!=typeof split)throw new Error("Invalid split in _split");const reSplit=new RegExp(split),limitInt=parseInt(limit);let strArray=limitInt>0?origStr.split(reSplit,limitInt):origStr.split(reSplit);if(!Array.isArray(strArray))return"";if(filter){const reFilter=new RegExp(filter);strArray=strArray.filter((item=>item.match(reFilter)))}const startInt=parseInt(start),endInt=parseInt(end);if(0===startInt&&0===endInt\|\|(strArray=0===endInt?strArray.slice(startInt):strArray.slice(startInt,endInt)),insertstr&&(strArray=strArray.map((item=>`${insertstr}${item}`))),"all"===resulttype)return strArray.join(join9);if("length"===resulttype)return String(strArray.length);if("index"===resulttype){const arrayIdx=strArray.findIndex((i=>i.trim()===val));return arrayIdx>=0?String(arrayIdx+1):""}return""}catch(err){return loginfo(err),""}}static _streq(origStr,funCfg){const{str:str}=funCfg;return"string"!=typeof str?"0":origStr===str?"1":"0"}static _substrAfter(origStr,funCfg){const{substr:substr,position:position,last:last,include:include}=funCfg,includeFlag=include,pos=parseInt(position);if("string"!=typeof substr)throw new Error("Invalid substr in _substrAfter");let index=0;return index=last?pos?origStr.lastIndexOf(substr,pos):origStr.lastIndexOf(substr):pos?origStr.indexOf(substr,pos):origStr.indexOf(substr),-1===index?origStr:String(includeFlag?origStr.substring(index):origStr.substring(index+substr.length))}static _substrBefore(origStr,funCfg){const{substr:substr,position:position,last:last,include:include}=funCfg,pos=parseInt(position),includeFlag=include;if("string"!=typeof substr)throw new Error("Invalid substr in substrBefore");let index=0;return index=last?pos?origStr.lastIndexOf(substr,pos):origStr.lastIndexOf(substr):pos?origStr.indexOf(substr,pos):origStr.indexOf(substr),-1===index?origStr:includeFlag?origStr.substring(0,index+substr.length):origStr.substring(0,index)}static _toBool(origStr){try{let strExp=origStr.replace(/true/g,"1");return strExp=strExp.replace(/false/g,"0"),strExp.search(/^[\s\d.+\-/()<>=!\|&:?]$/)>-1&&-1===strExp.search(/=>/)?String(Number(Boolean((0,eval)(strExp)))):(loginfo(`Invalid logic expression ${origStr} in _toBool`),!1)}catch(err){return!1}}static _toLowerCase(origStr){return origStr.toLowerCase()}static _toNum(origStr){let strExp=origStr;return strExp.search(/^[\s\d.+\-/()<>=!\|&:?]+$/)>-1&&-1===strExp.search(/=>/)&&strExp.trim().length>0?String(Number((0,eval)(strExp))):(loginfo(`Invalid arithmetic expression ${strExp} in _toNum`),"0")}static _toUpperCase(origStr){return origStr.toUpperCase()}static _trim(origStr){return origStr.trim()}static _trimEnd(origStr){return origStr.trimEnd()}static _trimStart(origStr){return origStr.trimStart()}static _uniq(origStr,funCfg){try{const{split:split,join:join9}=funCfg;if("string"!=typeof split\|\|"string"!=typeof join9)throw new Error("Invalid split in _uniq");const splitArr=origStr.split(new RegExp(split));return!Array.isArray(splitArr)\|\|splitArr.length<=1?origStr:[...new Set(splitArr)].join(join9)}catch(err){return loginfo(err),origStr}}static _urlencode(origStr,funCfg){try{const{method:method,encoding:encoding,percent:percent}=funCfg;if("URI"===method)return encodeURI(origStr);if("URIComponent"===method)return encodeURIComponent(origStr);if("iconv"===method){const str=iconv.encode(origStr,encoding).toString("hex").toUpperCase();if(!str\|\|str.length%2!=0)return logerr("Invalid str length in _urlencode"),"";if(percent){const matchResult=str.match(/../g);return matchResult?matchResult.map((item=>`%${item}`)).join(""):""}return str}return logerr(`Invalid method ${method} in _urlencode`),""}catch(err){return logerr(err),""}}static _urldecode(origStr,funCfg){try{const{method:method,encoding:encoding,percent:percent}=funCfg;if("URI"===method)return decodeURI(origStr);if("URIComponent"===method)return decodeURIComponent(origStr);if("iconv"===method){const re=percent?/.../g:/../g,matchResult=origStr.match(re);if(!matchResult)return"";const hexChars=matchResult.map((item=>parseInt(item.slice(1),16)));return iconv.decode(Buffer.from(hexChars),encoding)}return logerr(`Invalid method ${method} in _urlencode`),""}catch(err){return logerr(err),""}}static _urldelparams(origStr,funCfg){const{param1:param1,param2:param2,param3:param3}=funCfg;if("string"!=typeof param1)throw new Error("Invalid params in _urldel");let ret="";try{const url=new URL(origStr);if(!url\|\|!url.searchParams\|\|!url.searchParams.delete)return"";param1&&url.searchParams.delete(param1),param2&&url.searchParams.delete(param2),param3&&url.searchParams.delete(param3),ret=url.href}catch(err){ret=""}return ret\|\|""}static _urlget(origStr,funCfg){const{attrorpath:attrorpath,hashmethod:hashmethod,param:param,hostnameparts:hostnameparts,params:params,title:title,extname:extname,validextnames:validextnames}=funCfg;if("string"!=typeof attrorpath)throw new Error("Invalid attr in _urlget");let ret="";try{ret=["href","origin","protocol","username","password","host","search","hash","hostname","port","hrefwithoutsearch","param"].includes(attrorpath)?import_utils3.LsdUrl.getUrlAttribute(origStr,attrorpath,hostnameparts,param):import_utils3.LsdUrl.getFilePathFromUrl(origStr,attrorpath,hashmethod,hostnameparts,params,extname,validextnames,title)}catch(err){ret=""}return"string"==typeof ret?ret:""}static __setUrlParam(url,param,val,replaceFlag){if(!url\|\|!url.searchParams\|\|!url.searchParams.get\|\|"string"!=typeof param\|\|"string"!=typeof val)return!1;if(!param)return!0;if(!replaceFlag){if(null!==url.searchParams.get(param))return!0}return url.searchParams.set(param,val),!0}static _urlsetparams(origStr,funCfg){const{param1:param1,val1:val1,param2:param2,val2:val2,param3:param3,val3:val3,replace:replace}=funCfg,replaceFlag=replace;if("string"!=typeof param1)throw new Error("Invalid params in _urlset");let ret="";try{const url=new URL(origStr);_LsdString.__setUrlParam(url,param1,val1,replaceFlag),_LsdString.__setUrlParam(url,param2,val2,replaceFlag),_LsdString.__setUrlParam(url,param3,val3,replaceFlag),ret=url.href}catch(err){ret=""}return ret\|\|""}static _funPerformers={c_attr:_LsdString._c_attr,c_data:_LsdString._c_data,c_html:_LsdString._c_html,c_length:_LsdString._c_length,c_text:_LsdString._c_text,ceil:_LsdString._ceil,closingsubstr:_LsdString._closingsubstr,compress:_LsdString._compress,concat:_LsdString._concat,convertencoding:_LsdString._convertencoding,decode:_LsdString._decode,decompress:_LsdString._decompress,floor:_LsdString._floor,hashcode:_LsdString._hashcode,includes:_LsdString._includes,insert:_LsdString._insert,itemstoobj:_LsdString._itemstoobj,jsonparse:_LsdString._jsonparse,length:_LsdString._length,matchall:_LsdString._matchall,max:_LsdString._max,min:_LsdString._min,myfun:_LsdString.#myfun,numeq:_LsdString._numeq,numge:_LsdString._numge,numgt:_LsdString._numgt,numle:_LsdString._numle,numlt:_LsdString._numlt,notincludes:_LsdString._notincludes,padend:_LsdString._padEnd,padstart:_LsdString._padStart,parsefloat:_LsdString._parseFloat,parseint:_LsdString._parseInt,randomarryidxes:_LsdString._randomarryidxes,replace:_LsdString._replace,round:_LsdString._round,setcomplement:_LsdString._setcomplement,setdifference:_LsdString._setdifference,setintersection:_LsdString._setintersection,setunion:_LsdString._setunion,shuffle:_LsdString._shuffle,slice:_LsdString._slice,sort:_LsdString._sort,specialconvert:_LsdString._specialconvert,split:_LsdString._split,streq:_LsdString._streq,substrafter:_LsdString._substrAfter,substrbefore:_LsdString._substrBefore,tobool:_LsdString._toBool,tolowercase:_LsdString._toLowerCase,tonum:_LsdString._toNum,touppercase:_LsdString._toUpperCase,trim:_LsdString._trim,trimend:_LsdString._trimEnd,trimstart:_LsdString._trimStart,uniq:_LsdString._uniq,urldecode:_LsdString._urldecode,urldelparams:_LsdString._urldelparams,urlencode:_LsdString._urlencode,urlget:_LsdString._urlget,urlsetparams:_LsdString._urlsetparams}};function getTransformExpStrSync(origStr,funCfgs,defaultVal=""){return LsdString.transformStrSync(origStr,funCfgs,defaultVal)}async function getTransformExpStr(origStr,funCfgs,defaultVal="",myFunPerformers={}){return await LsdString.transformStr(origStr,funCfgs,defaultVal,myFunPerformers)}var fs2=__toESM(require("fs"),1),path2=__toESM(require("path"),1),import_header_generator=require("header-generator"),import_dateformat=__toESM(require("dateformat"),1),uuid2=__toESM(require("uuid"),1),import_utils4=require("@letsscrapedata/utils"),import_hpagent=require("hpagent"),download=__toESM(require("download"),1);async function lsdDownload(url,proxyUrl="",headers=null){const options={};if(proxyUrl){const proxyAgent=new import_hpagent.HttpsProxyAgent({keepAlive:!0,keepAliveMsecs:1e3,maxSockets:256,maxFreeSockets:256,scheduling:"lifo",proxy:proxyUrl});options.agent={https:proxyAgent}}let origContent;return headers&&(options.headers=headers),origContent=proxyUrl\|\|headers?await download.default(url,void 0,options):await download.default(url),origContent}var GetPara=class _GetPara{static getSpecialFunObj={};static setGetSpecialFunObj(getSpecialFunObj){return _GetPara.getSpecialFunObj=getSpecialFunObj,!0}static getOtherFunObj={};static setGetOtherFunObj(getOtherFunObj){return _GetPara.getOtherFunObj=getOtherFunObj,!0}static noNeedToReturnContent="";static async getPara(getCfg,otherInfo){if(!getCfg\|\|!getCfg.type\|\|!otherInfo)return"";const getType=getCfg.type;let content="";switch(getType){case"addedtaskparas":"function"==typeof _GetPara.getOtherFunObj.getAddedTaskParasFun&&(content=await _GetPara.getOtherFunObj.getAddedTaskParasFun(getCfg));break;case"datetime":content=_GetPara.getDatetime(getCfg);break;case"execedtaskparas":"function"==typeof _GetPara.getOtherFunObj.getExecedTaskParasFun&&(content=await _GetPara.getOtherFunObj.getExecedTaskParasFun(getCfg));break;case"file":content=await _GetPara.getFile(getCfg,otherInfo);break;case"httpheaders":content=await _GetPara.getHttpHeaders(getCfg,otherInfo);break;case"queuedcntwithparas":"function"==typeof _GetPara.getOtherFunObj.getQueuedCntWithParasCfgFun&&(content=await _GetPara.getOtherFunObj.getQueuedCntWithParasCfgFun(getCfg));break;case"queuedtaskparas":"function"==typeof _GetPara.getOtherFunObj.getQueuedTaskParasFun&&(content=await _GetPara.getOtherFunObj.getQueuedTaskParasFun(getCfg));break;case"random":content=_GetPara.getRandom(getCfg);break;case"response":content=await _GetPara.getResponse(getCfg,otherInfo);break;case"special":content=await _GetPara.getSpecial(getCfg,otherInfo);break;case"sysparas":content="please use ${sysParas.paraname}!!!";break;default:(0,import_utils4.unreachable)(getType)}return content}static async getParaInElement(getCfg,element,otherInfo){if(!getCfg\|\|!getCfg.type\|\|!otherInfo)return"";const getType=getCfg.type,paras=otherInfo.paras;let content="";if("screenshot"===getType)content=await _GetPara.getScreenshot(element,getCfg,paras);else(0,import_utils4.unreachable)(getType);return content}static async getParaInPage(getCfg,page,otherInfo){if(!getCfg\|\|!getCfg.type\|\|!otherInfo)return"";const getType=getCfg.type,paras=otherInfo.paras;let content="";switch(getType){case"content":content=await _GetPara.getContent(page);break;case"cookies":content="";break;case"mhtml":content=await _GetPara.getMhtml(page,getCfg,paras);break;case"pdf":content=await _GetPara.getPdf(page,getCfg,paras);break;case"screenshot":content=await _GetPara.getScreenshot(page,getCfg,paras);break;case"title":content=await _GetPara.getTitle(page,getCfg);break;case"window":content=await _GetPara.getWindow(page,getCfg);break;default:(0,import_utils4.unreachable)(getType)}return content}static async getContent(page){try{if(!page)return"";return await page.content()}catch(err){return""}}static async getCookies(page,getCfg){try{if(!page)return"";const{urls:urlsStr,domain:domain,name:name,path:path9,value:value}=getCfg;let content="";urlsStr.length>0&&urlsStr.split(";");let cookies=await page.cookies();return domain&&cookies.filter((c=>c.domain===domain)),name&&cookies.filter((c=>c.name===name)),path9&&cookies.filter((c=>c.path===path9)),content=value?cookies.length>0?cookies[0].value:"":JSON.stringify(cookies),content}catch(err){return""}}static getDatetime(getCfg){const{format:format}=getCfg;return(0,import_dateformat.default)(new Date,format)}static async getFile(getCfg,otherInfo){const paras=otherInfo.paras;try{const{url:url,pathtype:pathtype,hashmethod:hashmethod,hostnameparts:hostnameparts,params:params,extname:extname,validextnames:validextnames,basedir:basedir,setvar:setvar,pathvarname:pathvarname,proxy:proxyFlag,headers:headersStr,referer:referer}=getCfg;let pathStr=getCfg.path.trim();if(pathStr)pathStr=path2.join(pathStr);else{let filename=import_utils4.LsdUrl.getFilePathFromUrl(url,pathtype,hashmethod,Number(hostnameparts),params,extname,validextnames);pathStr=path2.join(basedir,filename)}if(pathStr){const subDir=path2.dirname(pathStr);if(!await(0,import_utils4.fsCheckOrCreateSubdir)(appConfig.cfg.baseFileDir,subDir))throw new Error("failed to checkOrCreateFileDir");pathStr=path2.join(appConfig.cfg.baseFileDir,pathStr)}const proxyUrl=proxyFlag&&otherInfo.proxyUrl?otherInfo.proxyUrl:"";let headers=null;(headersStr\|\|referer)&&(headers=headersStr?JSON.parse(headersStr):{},referer&&(headers.referer=referer));const origContent=await lsdDownload(url,proxyUrl,headers);return fs2.writeFileSync(pathStr,origContent),pathvarname&&paras&&!appConfig.RESERVED_PARA_NAMES.includes(pathvarname)&&(paras[pathvarname]=pathStr),setvar?origContent.toString("utf8"):_GetPara.noNeedToReturnContent}catch(err){throw new Error("error in get_file")}}static _genHeaderMethod={genLatestAndroid:import_header_generator.PRESETS.MODERN_ANDROID,genLatestDesktop:import_header_generator.PRESETS.MODERN_DESKTOP,genLatestLinux:import_header_generator.PRESETS.MODERN_LINUX,genLatestLinuxChrome:import_header_generator.PRESETS.MODERN_LINUX_CHROME,genLatestLinuxFirefox:import_header_generator.PRESETS.MODERN_LINUX_FIREFOX,genLatestMacos:import_header_generator.PRESETS.MODERN_MACOS,genLatestMacosChrome:import_header_generator.PRESETS.MODERN_MACOS_CHROME,genLatestMacosFirefox:import_header_generator.PRESETS.MODERN_MACOS_FIREFOX,genLatestMobile:import_header_generator.PRESETS.MODERN_MOBILE,genLatestWindows:import_header_generator.PRESETS.MODERN_WINDOWS,genLatestWindowsChrome:import_header_generator.PRESETS.MODERN_WINDOWS_CHROME,genLatestWindowsFirefox:import_header_generator.PRESETS.MODERN_WINDOWS_FIREFOX};static hasSameOrigin(sourceUrl,targetUrl,sourceType=""){try{const targetU=new URL(targetUrl),sourceU=new URL(sourceUrl);return targetU.origin===sourceU.origin\|\|!!["origStateHeaders","updatedStateHeaders"].includes(sourceType)&&targetU.hostname.endsWith(sourceU.hostname)}catch(err){return!1}}static async getHttpHeaders(getCfg,otherInfo){const{headermethod:headerMethod,origheaders:origHeaders}=getCfg,{genHeaderOptions:genHeaderOptions,newHttpHeaders:newHttpHeaders}=otherInfo,interceptionData=otherInfo.interceptionData;let headers={},secureHeadersFlag=!1,targetUrl="";try{if(_GetPara._genHeaderMethod[headerMethod]){secureHeadersFlag=!0;headers=new import_header_generator.HeaderGenerator(_GetPara._genHeaderMethod[headerMethod]).getHeaders()}else if("genNewByHttpOptions"===headerMethod){secureHeadersFlag=!0;headers=new import_header_generator.HeaderGenerator(genHeaderOptions).getHeaders()}else if("origheaders"===headerMethod)origHeaders&&(headers=JSON.parse(origHeaders));else{if(!interceptionData)throw new Error("logic error: no interceptionData");"cachedRequestHeaders"===headerMethod?(headers=Object.assign({},interceptionData.cachedRequestHeaders),targetUrl=interceptionData.cachedRequestUrl,interceptionData.updatedRequestUrl=targetUrl,interceptionData.updatedRequestHeaders=headers):"cachedResponseHeaders"===headerMethod?(headers=Object.assign({},interceptionData.cachedResponseHeaders),targetUrl=interceptionData.cachedResponseUrl,interceptionData.updatedResponseUrl=targetUrl,interceptionData.updatedResponseHeaders=headers):"origStateHeaders"===headerMethod?(headers=Object.assign({},interceptionData.origStateHeaders),targetUrl=interceptionData.origStateUrl,interceptionData.updatedStateUrl=targetUrl,interceptionData.updatedStateHeaders=headers):"updatedRequestHeaders"===headerMethod?(targetUrl=interceptionData.updatedRequestUrl,headers=interceptionData.updatedRequestHeaders):"updatedResponseHeaders"===headerMethod?(targetUrl=interceptionData.updatedResponseUrl,headers=interceptionData.updatedResponseHeaders):"updatedStateHeaders"===headerMethod?(targetUrl=interceptionData.updatedStateUrl,headers=interceptionData.updatedStateHeaders):logerr(`##get invalid httpMethod ${headerMethod} in getHttpHeaders`)}if(newHttpHeaders)for(const newHttpHeader of Object.values(newHttpHeaders)){const{name:name,source:source,value:value,deleteFlag:deleteFlag}=newHttpHeader;if(deleteFlag)delete headers[name];else if("value"===source)headers[name]=value;else if(!secureHeadersFlag&&source!==headerMethod){let sourceHeaders={},sourceUrl="";if("cachedRequestHeaders"===source)sourceUrl=interceptionData.cachedRequestUrl,sourceHeaders=interceptionData.cachedRequestHeaders;else if("cachedResponseHeaders"===source)sourceUrl=interceptionData.cachedResponseUrl,sourceHeaders=interceptionData.cachedResponseHeaders;else{if("origStateHeaders"!==source){logerr(`Invalid soure ${source} in httpheader`);continue}sourceUrl=interceptionData.origStateUrl,sourceHeaders=interceptionData.origStateHeaders}"string"==typeof sourceHeaders[name]&&_GetPara.hasSameOrigin(sourceUrl,targetUrl,source)?headers[name]=sourceHeaders[name]:value&&(headers[name]=value)}}return secureHeadersFlag?JSON.stringify(headers):""}catch(err){return logerr(err),""}}static async getMhtml(page,getCfg,paras){try{if(!page)return"";let pathStr=getCfg.path.trim();const{pathtype:pathtype,hashmethod:hashmethod,hostnameparts:hostnameparts,basedir:basedir,pathvarname:pathvarname}=getCfg;if(pathStr)pathStr=path2.join(pathStr);else{const urlStr=page.url();let title=await page.title();pathStr=import_utils4.LsdUrl.getFilePathFromUrl(urlStr,pathtype,hashmethod,Number(hostnameparts),"","mhtml","",title),pathStr=path2.join(basedir,pathStr)}const{data:data}=await page.sendCDPMessage("Page.captureSnapshot",{format:"mhtml"});if(pathStr){const subDir=path2.dirname(pathStr);if(!await(0,import_utils4.fsCheckOrCreateSubdir)(appConfig.cfg.baseFileDir,subDir))throw new Error("failed to checkOrCreateMhtmlDir");const fullName=path2.join(appConfig.cfg.baseFileDir,pathStr);fs2.writeFileSync(fullName,data),pathvarname&&paras&&!appConfig.RESERVED_PARA_NAMES.includes(pathvarname)&&(paras[pathvarname]=pathStr)}return String(data)}catch(err){throw logdbg(err),new Error("error in get_mhtml")}}static _getScreenSize(origSize){return String(Number(origSize))===origSize?`${origSize}px`:origSize}static async getPdf(page,getCfg,paras){try{if(!page)return"";const{pathtype:pathtype,hashmethod:hashmethod,hostnameparts:hostnameparts,basedir:basedir,scale:scale,displayheaderfooter:displayheaderfooter,headertemplate:headertemplate,footertemplate:footertemplate,printbackground:printbackground,landscape:landscape,pageranges:pageranges,format:format,width:width,height:height,top:top,right:right,bottom:bottom,left:left,onepage:onepage,hmargin:hmargin,pathvarname:pathvarname,setvar:setvar}=getCfg;let options={},pathStr=getCfg.path.trim();if(pathStr)pathStr=path2.join(pathStr);else{const urlStr=page.url();let title=await page.title();pathStr=import_utils4.LsdUrl.getFilePathFromUrl(urlStr,pathtype,hashmethod,Number(hostnameparts),"","pdf","",title),pathStr=path2.join(basedir,pathStr)}if(pathStr){const subDir=path2.dirname(pathStr);if(!await(0,import_utils4.fsCheckOrCreateSubdir)(appConfig.cfg.baseFileDir,subDir))throw new Error("failed to checkOrCreatePdfDir");pathStr=path2.join(appConfig.cfg.baseFileDir,pathStr),options.path=pathStr}if("1"!==scale&&(options.scale=Number(scale)),displayheaderfooter&&(options.displayHeaderFooter=displayheaderfooter),headertemplate&&(options.headerTemplate=headertemplate),footertemplate&&(options.footerTemplate=footertemplate),printbackground&&(options.printBackground=printbackground),landscape&&(options.landscape=landscape),pageranges&&(options.pageRanges=pageranges),top\|\|right\|\|bottom\|\|left\|\|displayheaderfooter){const margin={};top?margin.top=_GetPara._getScreenSize(top):displayheaderfooter&&!headertemplate&&(margin.top="60px"),bottom?margin.bottom=_GetPara._getScreenSize(bottom):displayheaderfooter&&!footertemplate&&(margin.bottom="60px"),right&&(margin.right=_GetPara._getScreenSize(right)),left&&(margin.left=_GetPara._getScreenSize(left)),options.margin=margin}if(onepage){const pageWidth=await page.pageWidth();let pageHeight=await page.pageHeight();hmargin&&(pageHeight+=Number(hmargin)),options.width=`${pageWidth}px`,options.height=`${pageHeight}px`,logdbg(`--------- pageHeight：${pageHeight}px`)}else width\|\|height?(width&&(options.width=_GetPara._getScreenSize(width)),height&&(options.height=_GetPara._getScreenSize(height))):"Letter"!==format&&(options.format=format);const buffer=await page.pdf(options);return pathvarname&&paras&&!appConfig.RESERVED_PARA_NAMES.includes(pathvarname)&&(paras[pathvarname]=pathStr),setvar?buffer.toString("base64"):_GetPara.noNeedToReturnContent}catch(err){throw logdbg(err),new Error("error in get_pdf")}}static getRandom(getCfg){try{const{min:minStr,max:maxStr}=getCfg,min=parseInt(minStr),max=parseInt(maxStr);if(min>=max)return"0";const rand=Math.round(Math.random()(max-min))+min;return String(rand)}catch(err){return"0"}}static async getResponse(getCfg,otherInfo){try{const{requrl:reqUrl,idx:idx,length:length}=getCfg;let responses=otherInfo?.interceptionData?.responses;if(!Array.isArray(responses)\|\|responses.length<=0\|\|!responses.every((r=>r.requestUrl)))return"";if(reqUrl){const re=new RegExp(reqUrl);responses=responses.filter((r=>r.requestUrl.match(re)))}const responsesCnt=responses.length;if(responsesCnt<=0)return"";if(length)return String(responsesCnt);const idxInt=parseInt(idx);let responseData="";return idxInt>=0&&idxInt<responsesCnt?responseData=responses[idxInt]?.responseData:idxInt<0&&idxInt+responsesCnt>=0&&(responseData=responses.slice(idxInt)[0]?.responseData),"string"==typeof responseData?responseData:""}catch(err){throw new Error("error in get_response")}}static async getScreenshot(pageOrElement,getCfg,paras){let options={};try{if(!pageOrElement)return"";let{basedir:basedir,path:pathStr,extname:extname,quality:quality,fullpage:fullpage,x:x,y:y,width:width,height:height,omitbackground:omitbackground,pathvarname:pathvarname,setvar:setvar}=getCfg;if(pathStr=pathStr.trim(),pathStr)pathStr=path2.join(basedir,pathStr);else if(pageOrElement.title){const title=await pageOrElement.title(),filename=`${import_utils4.LsdUrl.convertToValidPathname(title)}.${extname}`;pathStr=path2.join(basedir,filename)}else{const filename=`${uuid2.v4()}.${extname}`;pathStr=path2.join(basedir,filename)}if(pathStr){const subDir=path2.dirname(pathStr);if(!await(0,import_utils4.fsCheckOrCreateSubdir)(appConfig.cfg.baseFileDir,subDir))throw new Error("failed to checkOrCreatePdfDir");pathStr=path2.join(appConfig.cfg.baseFileDir,pathStr),options.path=pathStr}"jpeg"===extname&&(options.type="jpeg");const qualityInt=parseInt(quality);"jpeg"===extname&&qualityInt>=0&&(options.quality=qualityInt),fullpage&&(options.fullPage=fullpage);const widthInt=parseInt(width),heightInt=parseInt(height);if(widthInt>0&&heightInt>0){const clip={x:parseInt(x),y:parseInt(y),width:widthInt,height:heightInt};options.clip=clip}omitbackground&&(options.omitBackground=omitbackground);const buffer=await pageOrElement.screenshot(options);return pathvarname&&paras&&!appConfig.RESERVED_PARA_NAMES.includes(pathvarname)&&(paras[pathvarname]=pathStr),setvar&&buffer?buffer.toString("base64"):_GetPara.noNeedToReturnContent}catch(err){throw new Error("error in get_screenshot")}}static async getSpecial(getCfg,otherInfo){let content="";const method=getCfg.method;return"function"==typeof _GetPara.getSpecialFunObj[method]&&(content=await _GetPara.getSpecialFunObj[method](getCfg,otherInfo)),content}static async getTitle(page,getCfg){try{if(!page)return"";const{aspath:pathFlag}=getCfg,title=await page.title();return pathFlag?import_utils4.LsdUrl.convertToValidPathname(title):title}catch(err){return""}}static async getWindow(page,getCfg){try{const{key1:key1,key2:key2,key3:key3,key4:key4,key5:key5,key6:key6,keys:keysFlag,length:lengthFlag}=getCfg;if(!page\|\|!key1)return"";const keys=[key1,key2,key3,key4,key5,key6];let str=await page.windowMember(keys);if(!keysFlag&&!lengthFlag)return str;const obj=JSON.parse(str);return obj&&"object"==typeof obj?keysFlag?Array.from(Object.keys(obj)).join(","):lengthFlag&&Array.isArray(obj)?String(obj.length):"":""}catch(err){return logwarn(err),""}}},path3=__toESM(require("path"),1),fsp=__toESM(require("fs/promises"),1),import_utils5=require("@letsscrapedata/utils");async function responseCacheCb(response,options,pageData){if(options&&response)try{const{execData:execData,contentType:contentType,tabname:tabname,responseFlag:responseFlag,requestHeadersFlag:requestHeadersFlag,responseHeadersFlag:responseHeadersFlag,interceptionData:interceptionData}=options,{pageUrl:pageUrl,cookies:cookies}=pageData,request=response.request(),requestMethod=request.method(),requestUrl=request.url();let data=request.postData();const requestData=data\|\|"",responseData="text"===contentType?await response.text():"";if(tabname&&(execData[tabname]\|\|(execData[tabname]=[]),execData[tabname].push({pageUrl:pageUrl,requestMethod:requestMethod,requestUrl:requestUrl,requestData:requestData,responseData:responseData,size:String(responseData.length)})),responseFlag&&interceptionData.responses.push({pageUrl:pageUrl,requestMethod:requestMethod,requestUrl:requestUrl,requestData:requestData,responseData:responseData}),requestHeadersFlag){const requestHeaders=Object.assign({},request.allHeaders?await request.allHeaders():request.headers());!requestHeaders.cookie&&cookies&&(requestHeaders.cookie=cookies),interceptionData.cachedRequestUrl=requestUrl,interceptionData.cachedRequestHeaders=requestHeaders,logdbg("requestUrl: ",requestUrl),logdbg("requestHeaders: ",requestHeaders)}if(responseHeadersFlag){const responseHeaders=Object.assign({},response.allHeaders?await response.allHeaders():response.headers());interceptionData.cachedResponseUrl=requestUrl,interceptionData.cachedResponseHeaders=responseHeaders,logdbg("responseHeaders: ",responseHeaders)}}catch(err){logerr(err)}}var isHttpResponse=item=>void 0!==item.buffer;async function responseSaveCb(response,options,pageData){if(!options\|\|!response)return;const request=response.request(),requestUrl=request.url(),{execData:execData,tabname:tabname,pathType:pathType,hashMethod:hashMethod,baseDir:baseDir,hostnameParts:hostnameParts,params:params,extname:extname,validExtnames:validExtnames,encoding:encoding}=options,{pageUrl:pageUrl}=pageData;let filename=import_utils5.LsdUrl.getFilePathFromUrl(requestUrl,pathType,hashMethod,hostnameParts,params,extname,validExtnames);requestUrl.startsWith("http")\|\|logdbg("@@response: ",response),logdbg("@@reqUrl: ",requestUrl.slice(0,100)),logdbg("@@filename: ",filename);let pathStr=path3.join(baseDir,filename);if(!pathStr)return;const subDir=path3.dirname(pathStr);if(!await(0,import_utils5.fsCheckOrCreateSubdir)(appConfig.cfg.baseFileDir,subDir))throw new Error("failed to checkOrCreateReponseDir");const absolutePathStr=path3.join(appConfig.cfg.baseFileDir,pathStr),buffer=isHttpResponse(response)?await response.buffer():await response.body();let bufferEncoding=encoding;if(!encoding){const resourceType=request.resourceType();bufferEncoding=["document","stylesheet","script","xhr","fetch"].includes(resourceType)?"utf8":"binary"}if(await fsp.writeFile(absolutePathStr,buffer,bufferEncoding),tabname&&execData){const requestMethod=request.method();let data=request.postData();const record={pageUrl:pageUrl,requestMethod:requestMethod,requestUrl:requestUrl,requestData:data\|\|"",path:pathStr,size:String(buffer.length)};execData[tabname]?execData[tabname].push(record):execData[tabname]=[record]}}var import_node_fs=__toESM(require("fs"),1),import_utils7=require("@letsscrapedata/utils"),path4=__toESM(require("path"),1),fs3=__toESM(require("fs"),1),import_utils6=require("@letsscrapedata/utils"),solvers=[],addonFlag=!1;async function getScriptFromServer(scriptName){try{let content="",url=`${appConfig.cfg.urlPrefix}/script?name=${scriptName}`;const response=await fetch(url);if(200!==response.status)return"";const json=await response.json();return json&&0===json.code&&json.data&&"string"==typeof json.data?(content=Buffer.from(json.data,"base64").toString("utf8"),content):""}catch(err){return loginfo(err),""}}async function loadScriptIfNeeded(){if(addonFlag)return!0;addonFlag=!0;try{const addons=appConfig.cfg.lsdLaunchOptions.addons,addonPath=Array.isArray(addons)&&addons.length>0?addons[0]:"",captchas=appConfig.cfg.captcha.captchas;if(!addonPath\|\|!Array.isArray(captchas)\|\|0===captchas.length)return!0;let filename="";if(filename=path4.join(addonPath,"manifest.json"),!(0,import_utils6.isFileSync)(filename)){let manifestContent="ewogICJtYW5pZmVzdF92ZXJzaW9uIjogMywKICAibmFtZSI6ICJjYW1vdWZveCBhZGRvbiIsCiAgInZlcnNpb24iOiAiMS4wIiwKICAicGVybWlzc2lvbnMiOiBbCiAgICAic2NyaXB0aW5nIgogIF0sCiAgIndlYl9hY2Nlc3NpYmxlX3Jlc291cmNlcyI6IFsKICAgIHsKICAgICAgInJlc291cmNlcyI6IFsKICAgICAgICAic2NyaXB0cy8qIgogICAgICBdLAogICAgICAibWF0Y2hlcyI6IFsKICAgICAgICAiPGFsbF91cmxzPiIKICAgICAgXQogICAgfQogIF0sCiAgImNvbnRlbnRfc2NyaXB0cyI6IFsKICAgIHsKICAgICAgIm1hdGNoZXMiOiBbCiAgICAgICAgIjxhbGxfdXJscz4iCiAgICAgIF0sCiAgICAgICJqcyI6IFsKICAgICAgICAiaW5qZWN0LmpzIgogICAgICBdLAogICAgICAicnVuX2F0IjogImRvY3VtZW50X3N0YXJ0IgogICAgfQogIF0KfQ==";manifestContent=Buffer.from(manifestContent,"base64").toString("utf8"),fs3.writeFileSync(filename,manifestContent,"utf8")}if(filename=path4.join(addonPath,"inject.js"),!(0,import_utils6.isFileSync)(filename)){let injectContent="YXN5bmMgZnVuY3Rpb24gaW5qZWN0U2NyaXB0cygpIHsKICB0cnkgewogICAgZm9yIChjb25zdCBuYW1lIG9mIG5hbWVzKSB7CiAgICAgIHRyeSB7CiAgICAgICAgIGNvbnN0IHJlc3BvbnNlID0gYXdhaXQgZmV0Y2goY2hyb21lLnJ1bnRpbWUuZ2V0VVJMKGBzY3JpcHRzLyR7bmFtZX0uanNgKSk7CiAgICAgICAgY29uc3QgY29udGVudCA9IGF3YWl0IHJlc3BvbnNlLnRleHQoKTsKCiAgICAgICAgaWYgKGNvbnRlbnQpIHsKICAgICAgICAgIGNvbnN0IHNjcmlwdCA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoInNjcmlwdCIpOwogICAgICAgICAgc2NyaXB0LnRleHRDb250ZW50ID0gY29udGVudDsKICAgICAgICAgIGRvY3VtZW50LmRvY3VtZW50RWxlbWVudC5hcHBlbmRDaGlsZChzY3JpcHQpOwogICAgICAgICAgc2NyaXB0LnJlbW92ZSgpOwogICAgICAgIH0KICAgICAgfSBjYXRjaCAoZXJyKSB7CiAgICAgICAgY29uc29sZS5lcnJvcihgRmFpbGVkIHRvIGluamVjdCBzY3JpcHQgJHtuYW1lfTpgLCBlcnIpOwogICAgICB9CiAgICB9CiAgfSBjYXRjaCAoZXJyKSB7CiAgICBjb25zb2xlLmVycm9yKGVycik7CiAgfQp9CgppbmplY3RTY3JpcHRzKCk7Cg==";injectContent=Buffer.from(injectContent,"base64").toString("utf8");injectContent=`${`const names = ${JSON.stringify(captchas)};`}\n\n${injectContent}`,fs3.writeFileSync(filename,injectContent,"utf8")}if(filename=path4.join(addonPath,"scripts"),!await(0,import_utils6.fsCheckOrCreateDir)(filename,!0,!0))return logerr("Failed to create directory"),!1;for(const name of captchas){if(filename=path4.join(addonPath,"scripts",`${name}.js`),(0,import_utils6.isFileSync)(filename))continue;const content=await getScriptFromServer(name);content?fs3.writeFileSync(filename,content,"utf8"):loginfo(`Failed to get script of ${name}`)}return!0}catch(err){return loginfo(err),!1}}async function addCapatchaProvider(provider){return!(!provider\|\|"object"!=typeof provider\|\|"string"!=typeof provider?.name())&&(solvers.push(provider),await loadScriptIfNeeded(),!0)}function getSolver(captchaType){if(0===solvers.length)return null;for(const solver of solvers)if(solver.canSolveCaptcha(captchaType))return solver;return null}async function executeTask(task){const solver=getSolver(task.type);if(!solver)throw new Error(`##captcha no cpatha provider to solve ${task.type}`);let response;if(!solver[task.type])throw new Error("##captcha logic error");return response=await solver[task.type](task),response}var processors=new Map,captchaTypeAttr="data-captcha-type",widgetIdAttr="data-widget-id",captchaOptions={rootTag:"head",parentTag:"capwgts",captchaTag:"capwgt",solutionAttr:"value"};function checkCaptchaType(captchaWidget,captchaType){if(captchaWidget.captchaType!==captchaType)throw new Error(`##captcha invalid captchaType ${captchaWidget.captchaType}`);return!0}function isValidStringAttr(val){return"string"==typeof val&&!!val&&!["null","undefined"].includes(val)}async function getCaptchasOnce(page,msgFunc,options=captchaOptions){let captchaWidgets=[];if(msgFunc){const msgData={opType:"get"},res=await page.evaluate((async([funcName,data])=>{const func=window[funcName];return"function"==typeof func?func(data):""}),[msgFunc,msgData]);return Array.isArray(res)?(captchaWidgets=res,captchaWidgets):(logerr(`##captcha no function window.${msgFunc}`),[])}{const captchaSelector=`${options.rootTag} > ${options.parentTag} > ${options.captchaTag}`,captchaElements=await page.findElements(captchaSelector);if(0===captchaElements.length)return captchaWidgets;for(const captchaElement of captchaElements){const captchaWidget=await captchaElement.dataset();captchaWidgets.push(captchaWidget)}return captchaWidgets}}async function getWidgets(processor,page,msgFunc,options=captchaOptions){let captchaWidgets=await getCaptchasOnce(page,msgFunc,options);if(captchaWidgets.length>0)return captchaWidgets;const{needToRefresh:needToRefresh,maxRefreshTimes:maxRefreshTimes}=processor;if(needToRefresh&&maxRefreshTimes)for(let i=0;i<maxRefreshTimes;i++){if(!await needToRefresh(page))break;if(await page.reload(),await(0,import_utils7.sleep)(5e3),captchaWidgets=await getCaptchasOnce(page,msgFunc,options),captchaWidgets.length>0)return captchaWidgets}else if(await(0,import_utils7.sleep)(5e3),captchaWidgets=await getCaptchasOnce(page,msgFunc,options),captchaWidgets.length>0)return captchaWidgets;return[]}async function sendSolution(page,msgFunc,captchaWidget,solution,options=captchaOptions){const{captchaType:captchaType,widgetId:widgetId}=captchaWidget;if(msgFunc){const msgData={opType:"execute",captchaType:captchaType,widgetId:widgetId,solution:solution},res=await page.evaluate((async([funcName,data])=>{const func=window[funcName];return"function"==typeof func?func(data):""}),[msgFunc,msgData]);return"string"==typeof res?res:(logerr(`##captcha no function window.${msgFunc}`),"failed:noMsgFunc")}{const selector=`${options.rootTag} > ${options.parentTag} > ${options.captchaTag}[${captchaTypeAttr}="${captchaType}"][${widgetIdAttr}="${widgetId}"]`,captchaElement=await page.findElement(selector);if(captchaElement){return await captchaElement.setAttribute(options.solutionAttr,solution)?"ok":"failed"}return"failed:noElement"}}var amazonGetTask=function(captchaWidget){checkCaptchaType(captchaWidget,"amazon");const{captchaType:captchaType,pageurl:pageurl,sitekey:sitekey,iv:iv,context:context,challengeScript:challengeScript,captchaScript:captchaScript}=captchaWidget;return{type:captchaType,websiteURL:pageurl,websiteKey:sitekey,iv:iv,context:context,challengeScript:challengeScript,captchaScript:captchaScript}},amazonExecutionSolution=async function(page,msgFunc,captchaWidget,solution){const{existing_token:existing_token}=solution;return await sendSolution(page,msgFunc,captchaWidget,existing_token)},amazonProceesor={type:"amazon",findMethods:["intercept","hunt"],scriptName:"amazon",getTask:amazonGetTask,executeSolution:amazonExecutionSolution};processors.set("amazon",amazonProceesor);var geetestGetTask=function(captchaWidget){checkCaptchaType(captchaWidget,"geetest");const{captchaType:captchaType,version:version,url:url,gt:gt,challenge:challenge,apiServer:apiServer,captchaId:captchaId}=captchaWidget;if(!version\|\|!["v3","v4"].includes(version))throw new Error(`## captcha invalid version ${captchaWidget.version} in ${captchaWidget.captchaType}`);if(!url)throw new Error(`## captcha invalid url in ${captchaWidget.captchaType}`);if(!("v3"!==version\|\|gt&&challenge))throw new Error(`## captcha invalid gt or challenge in ${captchaWidget.captchaType} v3`);if("v4"===version&&!captchaId)throw new Error(`## captcha invalid captchaId in ${captchaWidget.captchaType} v4`);const task={type:captchaType,version:"v4"===version?4:3,websiteURL:url};return isValidStringAttr(gt)&&(task.gt=gt),isValidStringAttr(challenge)&&(task.challenge=challenge),isValidStringAttr(apiServer)&&(task.geetestApiServerSubdomain=apiServer),isValidStringAttr(captchaId)&&(task.captchaId=captchaId),task},geetestExecutionSolution=async function(page,msgFunc,captchaWidget,solution){checkCaptchaType(captchaWidget,"geetest");let solString="";if("v3"===captchaWidget.version){const{challenge:challenge,validate:validate,seccode:seccode}=solution,geetest_seccode=seccode\|\|`${validate}\|jordan`;if("string"!=typeof challenge\|\|!challenge)throw new Error(`##captcha invalid challenge in ${captchaWidget.captchaType}`);if("string"!=typeof validate\|\|!validate)throw new Error(`##captcha invalid validate in ${captchaWidget.captchaType}`);if("string"!=typeof seccode\|\|!seccode)throw new Error(`##captcha invalid seccode in ${captchaWidget.captchaType}`);solString=(0,import_utils7.encodeToBase64)({geetest_challenge:challenge,geetest_validate:validate,geetest_seccode:geetest_seccode})}else{if("v4"!==captchaWidget.version)return"failed:invalidVersion";{const{captcha_id:captcha_id,lot_number:lot_number,pass_token:pass_token,gen_time:gen_time,captcha_output:captcha_output}=solution;if("string"!=typeof captcha_id\|\|!captcha_id)throw new Error(`##captcha invalid captcha_id in ${captchaWidget.captchaType}`);if("string"!=typeof lot_number\|\|!lot_number)throw new Error(`##captcha invalid lot_number in ${captchaWidget.captchaType}`);if("string"!=typeof pass_token\|\|!pass_token)throw new Error(`##captcha invalid pass_token in ${captchaWidget.captchaType}`);if("string"!=typeof gen_time\|\|!gen_time)throw new Error(`##captcha invalid gen_time in ${captchaWidget.captchaType}`);if("string"!=typeof captcha_output\|\|!captcha_output)throw new Error(`##captcha invalid captcha_output in ${captchaWidget.captchaType}`);solString=(0,import_utils7.encodeToBase64)({captcha_id:captcha_id,lot_number:lot_number,pass_token:pass_token,gen_time:gen_time,captcha_output:captcha_output})}}return await sendSolution(page,msgFunc,captchaWidget,solString)},geetestProceesor={type:"geetest",findMethods:["intercept"],scriptName:"geetest",getTask:geetestGetTask,executeSolution:geetestExecutionSolution};processors.set("geetest",geetestProceesor);var recaptchaGetTask=function(captchaWidget,options){checkCaptchaType(captchaWidget,"recaptcha");const{captchaType:captchaType,version:version,url:url,sitekey:sitekey,action:action,s:s,enterprise:enterprise,invisible:invisible,userAgent:userAgent,cookies:cookies,apiDomain:apiDomain}=captchaWidget;if(!version\|\|!["v2","v3"].includes(version))throw new Error(`## captcha invalid version ${captchaWidget.version} in ${captchaWidget.captchaType}`);if(!url\|\|!sitekey)throw new Error(`## captcha invalid url or sitekey in ${captchaWidget.captchaType}`);let minScore=.3;"number"==typeof options?.minScore&&options.minScore>0&&options.minScore<1&&(minScore=options.minScore);const task={type:captchaType,version:"v3"===version?3:2,websiteURL:url,websiteKey:sitekey,isEnterprise:"true"===enterprise,isInvisible:"true"===invisible,minScore:minScore};return isValidStringAttr(action)&&(task.action=action),isValidStringAttr(s)&&(task.s=s),isValidStringAttr(userAgent)&&(task.userAgent=userAgent),isValidStringAttr(cookies)&&(task.cookies=cookies),isValidStringAttr(apiDomain)&&(task.apiDomain=apiDomain),task},recaptchaExecutionSolution=async function(page,msgFunc,captchaWidget,solution){checkCaptchaType(captchaWidget,"recaptcha");const{gRecaptchaResponse:gRecaptchaResponse}=solution;if("string"!=typeof gRecaptchaResponse\|\|!gRecaptchaResponse)throw new Error(`##captcha invalid gRecaptchaResponse in ${captchaWidget.captchaType}`);return await sendSolution(page,msgFunc,captchaWidget,gRecaptchaResponse)},recaptchaProceesor={type:"recaptcha",findMethods:["intercept","hunt"],scriptName:"recaptcha",getTask:recaptchaGetTask,executeSolution:recaptchaExecutionSolution};processors.set("recaptcha",recaptchaProceesor);var turnstileNeedToRefresh=async function(page){return 0===(await page.findElements("#cf-turnstile")).length},turnstileGetTask=function(captchaWidget){checkCaptchaType(captchaWidget,"turnstile");const{captchaType:captchaType,url:url,sitekey:sitekey,action:action,cData:cData,chlPageData:chlPageData}=captchaWidget,task={type:captchaType,websiteURL:url,websiteKey:sitekey};return isValidStringAttr(action)&&(task.action=action),isValidStringAttr(cData)&&(task.cData=cData),isValidStringAttr(chlPageData)&&(task.chlPageData=chlPageData),task},turnstileExecutionSolution=async function(page,msgFunc,captchaWidget,solution){checkCaptchaType(captchaWidget,"turnstile");const{token:token}=solution;if("string"!=typeof token\|\|!token)throw new Error(`##captcha invalid token in ${captchaWidget.captchaType}`);return await sendSolution(page,msgFunc,captchaWidget,token)},turnstileProceesor={type:"turnstile",findMethods:["intercept","hunt"],scriptName:"turnstile",getTask:turnstileGetTask,executeSolution:turnstileExecutionSolution,needToRefresh:turnstileNeedToRefresh,maxRefreshTimes:2};function getScriptFromLocal(scriptName){try{const filename=`${"./packages/scraper/tmp/lsd/inject"}/${scriptName}.js`;return import_node_fs.default.readFileSync(filename,"utf8")}catch(err){return""}}processors.set("turnstile",turnstileProceesor);var scriptMap=new Map;async function getScript(scriptName){let content=scriptMap.get(scriptName);if(content)return content;if(content=getScriptFromLocal(scriptName),content\|\|(content=await getScriptFromServer(scriptName)),"string"==typeof content&&content.length>20)return scriptMap.set(scriptName,content),content;throw new Error(`##captcha failed to get script of ${scriptName}`)}async function injectScripts(page,captchaTypes){let ret=!0;if("camoufox"===page.browserContext().browser().browserControllerType())return!0;if(!Array.isArray(captchaTypes)\|\|0===captchaTypes.length)return ret=!1,ret;const injectedScriptNames=[];for(const captchaType of captchaTypes){const processor=processors.get(captchaType);if(!processor)continue;if(!processor.findMethods.includes("intercept")\|\|!processor.scriptName)continue;if(injectedScriptNames.includes(processor.scriptName))continue;const scriptContent=await getScript(processor.scriptName);if(!scriptContent){ret=!1;break}injectedScriptNames.push(processor.scriptName);await page.addPreloadScript(scriptContent)\|\|(ret=!1)}return page.setCloseWhenFree(!0),ret}async function solveTokenCaptcha(page,captchaType,_proxy,options){const solveResult={code:0,captchaType:captchaType,taskResults:[]};loginfo(`######################### start to sovle ${captchaType} @${new Date}: `,options);let result="ok";try{const processor=processors.get(captchaType);if(!processor)return logerr(`##captcha cannot process ${captchaType}`),solveResult.code=1001,solveResult;const{getTask:getTask,executeSolution:executeSolution}=processor,captchaWidgets=await getWidgets(processor,page,"");if(0===captchaWidgets.length)return solveResult;const captchaWidget=captchaWidgets[0];captchaType!==captchaWidget.captchaType&&logerr(`##captcha different captchaType ${captchaType} ${captchaWidget.captchaType}`);const task=getTask(captchaWidget,options),{solution:solution,errorCode:errorCode,taskId:taskId,createTime:createTime,endTime:endTime}=await executeTask(task);if(solveResult.taskResults.push({solution:solution,errorCode:errorCode,taskId:taskId,elapse:endTime-createTime}),solution&&(result=await executeSolution(page,"",captchaWidget,solution),result.startsWith("ok")\|\|(solveResult.code=1003,solveResult.message=result),result)){const selector="button[type='submit']";await page.findElement(selector)}return loginfo(`######################### end to sovle ${captchaType} @${new Date}: `,solveResult),solveResult}catch(err){return logerr(err),solveResult.code=1010,solveResult.message=err.message,solveResult}}var import_node_fs2=__toESM(require("fs"),1),import_utils8=require("@letsscrapedata/utils"),processors2=new Map;async function getTextOfElement(page,loc,attr){if("pageUrl"===attr)return page.url();const element=await page.findElement(loc);return element?"textContent"===attr?await element.textContent():await element.attribute(attr):""}async function getBufferOfImage(page,loc,proxy,idx){const imgEle=await page.findElement(loc);if(!imgEle)return null;try{const src=await imgEle.attribute("src");if(src.toLowerCase().includes(".gif")){const pageUrl=page.url(),url=import_utils8.LsdUrl.getAbsoulteUrl(pageUrl,src),gifBuffer=await lsdDownload(url,proxy?.proxyUrl);return"number"==typeof idx&&import_node_fs2.default.writeFileSync(`./captchaImage${idx}.gif`,gifBuffer),gifBuffer}{"lazy"===await imgEle.attribute("loading")&&(await imgEle.scrollIntoView(),await(0,import_utils8.sleep)(2e3));const imgBuffer=await imgEle.screenshot();return"number"==typeof idx&&import_node_fs2.default.writeFileSync(`./captchaImage${idx}.png`,imgBuffer),imgBuffer}}catch(err){return logerr(err),null}}var textGetTask=async function(page,options,proxy=null){const{type:type,imageLocs:imageLocs,case:isCase,commentLoc:commentLoc,commentAttr:commentAttr,inputLoc:inputLoc,enter:enter,submitLoc:submitLoc,checkLoc:checkLoc,lang:lang,math:math,maxLength:maxLength,minLength:minLength,question:question,space:space,textType:textType}=options;if(!Array.isArray(imageLocs)\|\|0===imageLocs.length)throw new Error("invalidParas");if(!enter&&!submitLoc)throw new Error("invalidParas");if(!(!question\|\|commentLoc&&commentAttr))throw new Error("invalidParas");if(!checkLoc)throw new Error("invalidParas");const images=[];for(const[idx,imageLoc]of imageLocs.entries()){const buffer=await getBufferOfImage(page,imageLoc,proxy,idx);if(!buffer)throw new Error("noRequiredElement:imageLoc");images.push(buffer)}if(!await page.findElement(inputLoc))throw new Error("noRequiredElement:inputLoc");let comment="";if(commentLoc&&(comment=await getTextOfElement(page,commentLoc,commentAttr),!comment))throw new Error("noRequiredElement:commentLoc");if(submitLoc){if(!await page.findElement(submitLoc))throw new Error("noRequiredElement:submitLoc")}return{type:type,images:images,case:isCase,comment:comment,lang:lang,math:math,maxLength:maxLength,minLength:minLength,question:question,space:space,textType:textType,websiteUrl:page.url()}},textExecuteSolution=async function(page,options,solution){const{text:text}=solution,{inputLoc:inputLoc,enter:enter,submitLoc:submitLoc,checkLoc:checkLoc,checkAttr:checkAttr,failedStr:failedStr}=options,inputElement=await page.findElement(inputLoc);if(!inputElement)throw new Error("noRequiredElement");let succeeded=!1;if(enter){if(succeeded=await inputElement.input(text,{enter:enter}),!succeeded)return"failed:input"}else{const submitElement=await page.findElement(submitLoc);if(!submitElement)throw new Error("noRequiredElement");if(succeeded=await inputElement.input(text),!succeeded)return"failed:input";if(succeeded=await submitElement.click(),!succeeded)return"failed:submit"}await(0,import_utils8.sleep)(1e3);const checkElement=await page.findElement(checkLoc);if(!checkElement)return"ok";if(!checkAttr)return"failed:checkFailed-noCheckElement";const failedText="textContent"===checkAttr?await checkElement.textContent():await checkElement.attribute(checkAttr);return!failedStr&&failedStr.length>0?`failed:checkFailed-${failedText}`:failedText.includes(failedStr)?`failed:checkFailed-${failedStr}`:"ok"},textProcessor={type:"text",getTask:textGetTask,executeSolution:textExecuteSolution};processors2.set("text",textProcessor);var coordinateGetTask=async function(page,options,proxy=null){const{type:type,imageLocs:imageLocs,commentLoc:commentLoc,commentAttr:commentAttr,lang:lang,maxClicks:maxClicks,minClicks:minClicks}=options;if(!Array.isArray(imageLocs)\|\|0===imageLocs.length)throw new Error("invalidParas");if(minClicks<0\|\|maxClicks>0&&minClicks>=maxClicks)throw new Error("invalidParas");const images=[];for(const[idx,imageLoc]of imageLocs.entries()){const buffer=await getBufferOfImage(page,imageLoc,proxy,idx);if(!buffer)throw new Error("noRequiredElement:imageLoc");images.push(buffer)}let comment="";if(commentLoc&&(comment=await getTextOfElement(page,commentLoc,commentAttr),!comment))throw new Error("noRequiredElement:commentLoc");return{type:type,images:images,comment:comment,lang:lang,maxClicks:maxClicks,minClicks:minClicks}},coordinateExecuteSolution=async function(page,options,solution){const{coordinates:coordinates}=solution,{minClicks:minClicks,maxClicks:maxClicks,imageLocs:imageLocs,submitLoc:submitLoc,checkLoc:checkLoc,checkAttr:checkAttr,failedStr:failedStr}=options;if(!Array.isArray(coordinates)\|\|coordinates.length<minClicks\|\|maxClicks>0&&coordinates.length>maxClicks)return"failed:invalidSolution";let succeeded=!1;const imageElement=await page.findElement(imageLocs[0]);if(!imageElement)throw new Error("noRequiredElement");const boundingBox=await imageElement.boundingBox();if(!boundingBox)return"failed:click";const{x:startX,y:startY}=boundingBox;for(const{x:x,y:y}of coordinates){if(succeeded=await page.mouseClick(startX+x,startY+y),!succeeded)return"failed:click";await(0,import_utils8.sleep)(200)}if(await(0,import_utils8.sleep)(1e3),submitLoc){const submitElement=await page.findElement(submitLoc);if(!submitElement)throw new Error("noRequiredElement");if(succeeded=await submitElement.click(),!succeeded)return"failed:submit";await(0,import_utils8.sleep)(1e3)}const checkElement=await page.findElement(checkLoc);if(!checkElement)return"ok";if(!checkAttr)return"failed:checkFailed-noCheckElement";const failedText="textContent"===checkAttr?await checkElement.textContent():await checkElement.attribute(checkAttr);return!failedStr&&failedStr.length>0?`failed:checkFailed-${failedText}`:failedText.includes(failedStr)?`failed:checkFailed-${failedStr}`:"ok"},coordinateProcessor={type:"text",getTask:coordinateGetTask,executeSolution:coordinateExecuteSolution};async function solveRecognitionCaptcha(page,options,proxy,tryTimes=1){const captchaType=options.type;loginfo(`######################### start to sovle ${captchaType} @${new Date}: `,options);const solveResult={code:0,captchaType:captchaType,taskResults:[]};let result="";try{const processor=processors2.get(captchaType);if(!processor)return logerr(`##captcha cannot process ${captchaType}`),solveResult.code=1001,solveResult;const{getTask:getTask,executeSolution:executeSolution}=processor;for(let i=1;i<=tryTimes;i++){const task=await getTask(page,options,proxy),{solution:solution,errorCode:errorCode,taskId:taskId,createTime:createTime,endTime:endTime}=await executeTask(task),taskResult={solution:solution,errorCode:errorCode,taskId:taskId,elapse:endTime-createTime};if(solveResult.taskResults.push(taskResult),solution){if(result=await executeSolution(page,options,solution),result.startsWith("ok"))return solveResult;if(result.startsWith("failed:checkFailed")){taskResult.errorCode="ERROR_CHECK_FAILED";continue}taskResult.errorCode="ERROR_SOLUTION_FAILED";break}}return solveResult.code=1003,solveResult.message=result,solveResult}catch(err){logerr(err);const message=err.message;return"invalidParas"===message?solveResult.code=1001:message.startsWith("noRequiredElement")?solveResult.code=1004:solveResult.code=1010,solveResult.message=message,loginfo(`######################### end to sovle ${captchaType} @${new Date}: `,solveResult),solveResult}}processors2.set("coordinate",coordinateProcessor);var uuid3=__toESM(require("uuid"),1);function calcCapCredits(credits){return credits>0?credits>>appConfig.GEN_CREDITS_BITS:0}function calcGenCredits(credits){return credits>0?credits&appConfig.GEN_CREDITS_AND:0}var TaskScraper=class extends import_node_events.default{#template;#actionConfigs;#fontsConfig;_fontsConfig;#myfunScripts;#version;#templateId;_templateId;#taskId;_taskId;#captchaOptions;#loginOptions;#defaultElementSource;#splitStr;#skipActions;#extractColumns;#performOneTask;#taskContext;_taskContext;#paras;_paras;#parasStr;#interceptionData;#actionPerformers;#errObj;#lastUrl;constructor(templateConfig,inParas,taskNetworkContext,taskOptions={}){super();const{template:template,actionConfigs:actionConfigs,fontsConfig:fontsConfig,myfunScripts:myfunScripts}=templateConfig,{proxy:proxy=null,page:page=null,browserApiContext:browserApiContext=null,standaloneApiContext:standaloneApiContext=null}=taskNetworkContext,{performOneTask:performOneTask2=appConfig.performOneTask,taskId:taskId=0,skipActions:skipActions=[],extractColumns:extractColumns=[],version:version="1.0",splitStr:splitStr="::",authInfo:authInfo={}}=taskOptions,origStateData=taskOptions.stateData?taskOptions.stateData:{cookies:[],localStorage:[],headers:{},userData:{}},defaultElementSource=taskOptions.defaultElementSource?taskOptions.defaultElementSource:template.defaultElementSource;try{this.#template=template,this.#actionConfigs=actionConfigs,this.#fontsConfig=fontsConfig,this._fontsConfig=fontsConfig,this.#myfunScripts=myfunScripts,this.#version=version,this.#templateId=template.templateId,this._templateId=template.templateId,this.#taskId=taskId,this._taskId=taskId,this.#captchaOptions=taskOptions.captchaOptions?taskOptions.captchaOptions:{captchaTypes:[]},this.#loginOptions=taskOptions.loginOptions?taskOptions.loginOptions:{loginWhens:[],pageMatchInfos:[]},this.#defaultElementSource=defaultElementSource,this.#skipActions=skipActions,this.#extractColumns=extractColumns,this.#splitStr=splitStr,this.#performOneTask=performOneTask2,this.#taskContext={proxy:proxy,page:page,browserApiContext:browserApiContext,standaloneApiContext:standaloneApiContext,stateApiContext:browserApiContext\|\|standaloneApiContext,taskApiContext:null,popupPage:null,browserElesStack:[],cheerioPage:null,cheerioElesStack:[],loopsStack:[],continueFlag:!1,breakFlag:!1,origStateData:origStateData,newStateData:null,execData:{},subtasks:[],credits:0,subtaskResults:[]};const sysParas=new SysParas(this.#taskContext,this.#templateId,this.#taskId,this.#version);this.#paras={inParas:inParas,sysParas:sysParas,authInfo:authInfo,userData:Object.assign({},origStateData.userData),responses:[]},this.#parasStr=this.#getParasStr();const origStateUrl=origStateData.localStorage.length>0?origStateData.localStorage[0].origin:origStateData.cookies.length>0?`https://${origStateData.cookies[0].domain}`:"";this.#interceptionData={responses:[],cachedRequestUrl:"",cachedRequestHeaders:{},cachedResponseUrl:"",cachedResponseHeaders:{},updatedRequestUrl:"",updatedRequestHeaders:{},updatedResponseUrl:"",updatedResponseHeaders:{},origStateUrl:origStateUrl,origStateHeaders:origStateData.headers,updatedStateUrl:"",updatedStateHeaders:{}},this.#actionPerformers={api:this.#performApiAction,break:this.#performBreakAction,captcha:this.#performCaptchaAction,click:this.#performClickAction,continue:this.#performContinueAction,exit:this.#performExitAction,extract:this.#performExtractAction,goto:this.#performGotoAction,hover:this.#performHoverAction,ifelse:this.#performIfelseAction,input:this.#performInputAction,intercept:this.#performInterceptAction,loopdowhile:this.#performLoopdowhileAction,loopfor:this.#performLoopforAction,loopineles:this.#performLoopinelesAction,loopinstr:this.#performLoopinstrAction,misc:this.__performMiscAction,scroll:this.#performScrollAction,select:this.#performSelectAction,setvar:this.#performSetvarAction,subtask:this.#performSubtaskAction,wait:this.#performWaitAction},this.#errObj={normal:-1,parasinvalid:-2,accountinvalid:-41,accountlocked:-42,loginfailed:-43,loginneeded:-49,tologin:-40,accessforbidden:-61,cfginvalid:-69,captchafailed:-52,switchproxynow:-86,switchproxyfuture:-87,other:-98,logicerror:-98,tetimeout:-79},this.#lastUrl="",this._taskContext=this.#taskContext,this._paras=this.#paras,this.#fontsConfig&&logdbg(this.#fontsConfig)}catch(err){throw logerr("##te Exeception in ScraperTask.constructor"),new Error(err)}}#getElementSource(eleCfg){const origElementSource=this.#getCfgStrAttr(eleCfg,"source");return"default"===origElementSource?this.#defaultElementSource:"browser"===origElementSource?"browser":"cheerio"===origElementSource?"cheerio":void(0,import_utils9.unreachable)(origElementSource)}#setFailedCredits(credits){this.#taskContext.credits=credits,logwarn(new Error)}#getActionName(actionType){return`action${actionType[0].toUpperCase()}${actionType.slice(1)}`}#canResetPage(page,actionType){const{browserElesStack:browserElesStack,cheerioElesStack:cheerioElesStack}=this.#taskContext;return!(!page\|\|browserElesStack.length>0\|\|cheerioElesStack.length>0)\|\|(this.#procErrname("cfginvalid",this.#getActionName(actionType),"resetPage"),!1)}#procErrname(errName,scenarioName,reason){"ignore"!==errName&&logwarn(`##tmp procErrname eName ${errName} scenario ${scenarioName} reason ${reason} tId ${this.#templateId} kId ${this.#taskId} p ${this.#parasStr}`);let credits=-69;if("string"!=typeof errName\|\|!errName\|\|!scenarioName)throw this.#taskContext.credits=credits,new Error("##te invalid paras in _procErrname");if("ignore"===errName)return!0;throw void 0!==this.#errObj[errName]?(credits=this.#errObj[errName],this.#taskContext.credits=credits,new Error(`##te exit with errname ${errName} in ${scenarioName}`)):(this.#taskContext.credits=credits,new Error(`##te invalid errName ${errName} in ${scenarioName}`))}_procErrname(errName,scenarioName,reason){return this.#procErrname(errName,scenarioName,reason)}#getParasStr(){if(this.#paras.inParas&&"object"==typeof this.#paras.inParas){const allParaNames=Array.from(Object.keys(this.#paras.inParas)),paraNames=allParaNames.filter((n=>n.startsWith("para"))),aparaNames=allParaNames.filter((n=>n.startsWith("apara"))),rparaNames=allParaNames.filter((n=>n.startsWith("rpara")));let parasStr=paraNames.map((n=>this.#paras.inParas[n])).join(this.#splitStr);return(aparaNames.length>0\|\|rparaNames.length>0)&&(parasStr=`${parasStr}@@${aparaNames.map((n=>this.#paras.inParas[n])).join(this.#splitStr)}`,rparaNames.length>0&&(parasStr=`${parasStr}@@${rparaNames.map((n=>this.#paras.inParas[n])).join(this.#splitStr)}`)),parasStr}return""}#setCheerioPage(htmlOrXml,isHtml=!0){return this.#taskContext.cheerioPage?this.#taskContext.cheerioPage.load(htmlOrXml,isHtml):this.#taskContext.cheerioPage=new import_controller3.CheerioPage(htmlOrXml,isHtml),!0}#setNormalVariable(varName,varValue){if("string"!=typeof varName\|\|"string"!=typeof varValue\|\|appConfig.RESERVED_PARA_NAMES.includes(varName)?this.#procErrname("cfginvalid","setNormalVar","invalidParas"):"cheerioHtml"!==varName&&"cheerioXml"!==varName\|\|(this.#taskContext.cheerioElesStack.length>0&&this.#procErrname("cfginvalid","setNormalVar","cheerioElesStack"),this.#setCheerioPage(varValue,"cheerioHtml"===varName)),varName.startsWith("userData.")){const newVarName=varName.slice(9);0===newVarName.length&&this.#procErrname("cfginvalid","setNormalVar","invalidUserDataVarName"),this.#paras.userData[newVarName]=varValue}else if(varName.startsWith("inParas_")){const newVarName=varName.slice(8);0!==newVarName.length&&void 0!==this.#paras.inParas[newVarName]\|\|this.#procErrname("cfginvalid","setNormalVar","invalidInParasVarName"),this.#paras.inParas[newVarName]=varValue,this.#parasStr=this.#getParasStr()}else this.#paras[varName]=varValue;return(varName.startsWith("dbgdbg")\|\|varName.startsWith("cx")&&varName.match(/^cx\d$/))&&logwarn(`##dbg setvar ${varName}: ${varValue}`),!0}#validateUrl(url){if("string"!=typeof url)return!1;const lowerUrl=url.toLowerCase();return lowerUrl.startsWith("https://")\|\|lowerUrl.startsWith("http://")}#validateCfgType(cfg){if(cfg?.[appConfig.XML_ATTR_KEY]?.type)return!0;throw new Error("##te Invalid element type config(validate)")}#transTemplToStr(templStr,paras){if("string"!=typeof templStr)throw new Error(`##te Invalid templStr: ${templStr}`);if(templStr.indexOf("${")<0)return templStr;try{return function(pars){const names=Object.keys(pars),vals=Object.values(pars);return new Function(...names,`return \`${this}\`;`)(...vals)}.call(templStr,paras)}catch(error){return""}}#getCfgStrAttr(cfg,cfgAttrName,transTemplFlag=!0,mustFlag=!0){if(!cfg\|\|"string"!=typeof cfgAttrName)throw new Error(`##te Invalid config or attribute, attribute: ${cfgAttrName}`);const attrVal=cfg[appConfig.XML_ATTR_KEY][cfgAttrName];if(null===attrVal)return attrVal;if(void 0===attrVal){if(mustFlag)throw new Error(`##te Config has no attribute ${cfgAttrName}`);return""}return!transTemplFlag\|\|["type","name","version"].includes(cfgAttrName)?attrVal:this.#transTemplToStr(attrVal,this.#paras)}_getCfgStrAttr(cfg,cfgAttrName,transTemplFlag=!0,mustFlag=!0){return this.#getCfgStrAttr(cfg,cfgAttrName,transTemplFlag,mustFlag)}#getCfgBoolAttr(cfg,cfgAttrName){if(!cfg)throw new Error(`##te Invalid action config or attribute, attribute: ${cfgAttrName}`);const attrVal=cfg[appConfig.XML_ATTR_KEY][cfgAttrName];if("boolean"!=typeof attrVal)throw loginfo(`##te attrNames: ${JSON.stringify(cfg[appConfig.XML_ATTR_KEY])}, attrName: ${cfgAttrName}`),new Error(`##te Element has no bool attribute ${cfgAttrName} or it is undefined/null`);return attrVal}_getCfgBoolAttr(cfg,cfgAttrName){return this.#getCfgBoolAttr(cfg,cfgAttrName)}#getCfgTextContext(cfg,mustFlag=!0){if(!cfg)throw new Error("##te Invalid action config or attribute");const attrVal=cfg[appConfig.XML_CHAR_KEY];if("string"==typeof attrVal)return attrVal;if(mustFlag)throw new Error("##te Config has no textContext");return""}_getCfgTextContext(cfg,mustFlag=!0){return this.#getCfgTextContext(cfg,mustFlag)}#getCfgChildrenOf(parentCfg,cfgType,mustFlag=!1){try{const cKey=appConfig.XML_CHILDREN_KEY,aKey=appConfig.XML_ATTR_KEY;let ret=parentCfg[cKey]?parentCfg[cKey][cfgType]:null;if(Array.isArray(ret))return ret;if(void 0===ret){const retObj={};return"elecontent"===cfgType?(retObj[aKey]={type:"text",multieles:!0,trim:!0,join:"\n",line:!1,top:!1},[retObj]):"ocrcfg"===cfgType?(retObj[aKey]={type:"api",nlchars:"",seperator:!1,lang:"zh"},[retObj]):[]}if(mustFlag)throw new Error(`##te Invalid cfgType ${cfgType} children of parentCfg ${JSON.stringify(parentCfg)}`);return[]}catch(err){return logerr(err),[]}}#generateElementCfg(loc,source="default",absolute=!1){const cfg={};return cfg[appConfig.XML_ATTR_KEY]={loc:loc,source:source,absolute:absolute},cfg}_getCfgChildrenOf(parentCfg,cfgType,mustFlag=!1){return this.#getCfgChildrenOf(parentCfg,cfgType,mustFlag)}#getOptionalCfgChildOf(parentCfg,cfgType){const cfgChildren=this.#getCfgChildrenOf(parentCfg,cfgType);return cfgChildren.length>0?cfgChildren[0]:null}#getRequiredCfgChildOf(parentCfg,cfgType){const cfgChildren=this.#getCfgChildrenOf(parentCfg,cfgType);if(0===cfgChildren.length)throw new Error(`##te There is no ${cfgType} tag config`);return cfgChildren[0]}#getPageInfo(){if(this.#taskContext.page)return this.#taskContext.page.pageInfo();throw new Error("Cannot get pageInfo from null browser page")}#isHeadless(){if(this.#taskContext.page)return this.#taskContext.page.browserContext().browser().isHeadless();throw new Error("Cannot get headless from null browser page")}#getIntVal(mathExprStr,roundType="ceil"){if("string"!=typeof mathExprStr)throw new Error(`##te Inavlid math expr: ${mathExprStr}`);if(!["ceil","round","floor"].includes(roundType))throw new Error(`##te Inavlid round type: ${roundType}`);return Number(getTransformExpStrSync(mathExprStr,[{type:"tonum"},{type:roundType}]))}#getBoolVal(boolExprStr){if("string"!=typeof boolExprStr)throw new Error(`##te Inavlid bool expr: ${boolExprStr}`);return Boolean(Number(getTransformExpStrSync(boolExprStr,[{type:"tobool"}])))}async#processWait(waitMsStr){let waitMsInt=parseInt(waitMsStr);return waitMsInt>0&&await(0,import_utils9.sleep)(waitMsInt),!0}#addTaskCredits(actCfg,genFlag=!0,capFlag=!0,inGenCredits=1,inCapCredits=1){const gencreditsFlag=actCfg?this.#getCfgBoolAttr(actCfg,"gen"):genFlag,capcreditsFlag=actCfg?this.#getCfgBoolAttr(actCfg,"cap"):capFlag,genCredits=gencreditsFlag?inGenCredits:0,capCredits=this.#template.capName&&capcreditsFlag?inCapCredits:0;return loginfo(`##credit addTaskCredits: gen ${genCredits}, cap ${capCredits}`),this.#taskContext.credits+=(capCredits<<appConfig.GEN_CREDITS_BITS)+genCredits,!0}#getDataLoginCaptchaInfo(actCfg,actionType){if(!["click","goto","input","scroll","select"].includes(actionType))throw new Error(`logic error actionType ${actionType} cannot call this function`);const{popuppage:popupPageType,wait:waitStr,errname:errName,eurl:eurl,eloc:eloc,pn1:pn1,pv1:pv1,pn2:pn2,pv2:pv2}=this.#transformStrAttrsInCfg(actCfg,!1),loginFlag=this.#getCfgBoolAttr(actCfg,"login")&&!!this.#captchaOptions.pageMatchInfos,captchaFlag=this.#getCfgBoolAttr(actCfg,"captcha")&&!!this.#loginOptions.pageMatchInfos,dataPageMatchInfo=eurl\|\|eloc\|\|pn1\|\|pv1\|\|pn2\|\|pv2?{eurl:eurl,eloc:eloc,pn1:pn1,pv1:pv1,pn2:pn2,pv2:pv2}:null;if("ignored"===popupPageType\|\|!loginFlag&&!captchaFlag&&!dataPageMatchInfo)return null;const popupSubtaskFlag=this.#getCfgBoolAttr(actCfg,"popupsubtask");let waitMs=parseInt(waitStr);waitMs>0\|\|(waitMs=0);const popupFlag="ignored"!==popupPageType,popupPageFlags={captchaFlag:popupFlag&&captchaFlag&&!!this.#captchaOptions?.captchaWhens?.includes("popup"),loginFlag:popupFlag&&loginFlag&&!!this.#loginOptions?.loginWhens?.includes("popup"),taskFlag:!(popupFlag&&popupSubtaskFlag)};if(popupFlag&&!popupPageFlags.captchaFlag&&!popupPageFlags.loginFlag&&!dataPageMatchInfo)return this.#procErrname("cfginvalid",this.#getActionName(actionType),"invalidPopupCfg"),null;let currentFlag="ignored"===popupPageType;const currentPageFlags={captchaFlag:captchaFlag&&!!this.#captchaOptions?.captchaWhens?.includes("current"),loginFlag:loginFlag&&!!this.#loginOptions?.loginWhens?.includes("current"),taskFlag:currentFlag&&!!dataPageMatchInfo};if(currentFlag&&!currentPageFlags.captchaFlag&&!currentPageFlags.loginFlag&&currentPageFlags.taskFlag)return this.#procErrname("cfginvalid",this.#getActionName(actionType),"invalidCurrentCfg"),null;return{actionType:actionType,errName:errName,dataPageMatchInfo:dataPageMatchInfo,waitMs:waitMs,popupFlag:popupFlag,popupPageType:popupPageType,popupPageFlags:popupPageFlags,currentFlag:currentFlag,currentPageFlags:currentPageFlags,captchaPageMatchInfos:this.#captchaOptions.pageMatchInfos,loginPageMatchInfos:this.#loginOptions.pageMatchInfos}}async#getPageMatchInfo(page,pageMatchInfos,checkLocFlag=!0){const urlStr=page.url();if(!urlStr.trim())return null;const url=new URL(urlStr),href=url.href.toLowerCase();for(const pmi of pageMatchInfos){if(!pmi.eurl&&!pmi.pn1&&!pmi.pn2)return null;let urlFlag=!pmi.eurl;if(pmi.eurl){const urls=pmi.eurl.split(",");urlFlag=href.startsWith(urls[0])&&urls.slice(1).every((item=>href.includes(item)))}const p1Flag=!pmi.pn1\|\|pmi.pn1&&url.searchParams.get(pmi.pn1)===pmi.pv1,p2Flag=!pmi.pn2\|\|pmi.pn2&&url.searchParams.get(pmi.pn2)===pmi.pv2;if(!urlFlag\|\|!p1Flag\|\|!p2Flag)continue;let locFlag=!0;if(pmi.eloc&&checkLocFlag)try{for(let i=0;i<30;i++){if((await page.findElements(pmi.eloc)).length>0){locFlag=!0;break}await(0,import_utils9.sleep)(2e3)}}catch(err){locFlag=!1}if(locFlag)return pmi}return null}async#processPageLoginCaptcha(page,dataLoginCatchaInfo,scenario){const{actionType:actionType,dataPageMatchInfo:dataPageMatchInfo,popupPageType:popupPageType,errName:errName}=dataLoginCatchaInfo,pageFlags="popup"===scenario?dataLoginCatchaInfo.popupPageFlags:dataLoginCatchaInfo.currentPageFlags,{captchaFlag:captchaFlag,loginFlag:loginFlag,taskFlag:taskFlag}=pageFlags;if(captchaFlag&&dataLoginCatchaInfo.captchaPageMatchInfos){if(await this.#getPageMatchInfo(page,dataLoginCatchaInfo.captchaPageMatchInfos))return!0}if(loginFlag&&dataLoginCatchaInfo.loginPageMatchInfos){if(await this.#getPageMatchInfo(page,dataLoginCatchaInfo.loginPageMatchInfos))return this.#procErrname("tologin",this.#getActionName(actionType),scenario),!0}if(dataPageMatchInfo){if(await this.#getPageMatchInfo(page,[dataPageMatchInfo]))return"popup"===scenario&&(taskFlag?(this.#canResetPage(page,actionType),this.#taskContext.page=page):this.#taskContext.popupPage=page),!0;if("required"===popupPageType)return this.#procErrname(errName,this.#getActionName(actionType),"noMatchedPopupPage"),!1}return!0}async processNewPageLoginCaptcha(dataLoginCaptchaInfo,pageOperateFun,pageOperateArgs){const{actionType:actionType,errName:errName,waitMs:waitMs,popupPageType:popupPageType}=dataLoginCaptchaInfo,popupTimeout=appConfig.CHECK_POPUP_PAGE_TIMEOUT1;setTimeout((()=>this.emit(appConfig.EVENT_TYPES.lsdScraperPopup,{code:1009,message:"timeout",data:null})),popupTimeout),await pageOperateFun(...pageOperateArgs),waitMs&&await(0,import_utils9.sleep)(waitMs);const[response]=await(0,import_node_events2.once)(this,appConfig.EVENT_TYPES.lsdScraperPopup);loginfo(`##tmp Taskscraper received popup event ${response.code}`);const popupPage=response&&0===response.code&&response.data?response.data:null;if(!popupPage){if("required"!==popupPageType)return!0;this.#procErrname(errName,this.#getActionName(actionType),"noPopupPage")}return await this.#processPageLoginCaptcha(popupPage,dataLoginCaptchaInfo,"popup"),!0}#checkBrowserElementSource(elementSource,actionName){"browser"===elementSource&&this.#taskContext.page\|\|this.#procErrname("cfginvalid",this.#getActionName(actionName),"invalidBrowserElementSource")}async#getApiContxt(context){if("browser"===context)return this.#taskContext.browserApiContext;if("standalone"===context)return this.#taskContext.standaloneApiContext;if("state"===context)return this.#taskContext.stateApiContext;if("task"===context){if(!this.#taskContext.taskApiContext){const options=this.#taskContext.proxy?{proxy:this.#taskContext.proxy}:{};this.#taskContext.taskApiContext=await import_controller3.controller.newApiContext(options)}return this.#taskContext.taskApiContext}return null}#getApiRequestHeaders(reqUrl,headersSource,headersStr,referer,requestCfgs){let headers={};const headerCfgs=requestCfgs.filter((requestCfg=>"header"===this.#getCfgStrAttr(requestCfg,"type")));try{if("headers"!==headersSource\|\|"headers"===headersSource&&headersStr\|\|referer\|\|headerCfgs.length>0){let sourceUrl="";const interceptionData=this.#interceptionData;if("headers"===headersSource&&headersStr?headers=JSON.parse(headersStr):"cachedRequestHeaders"===headersSource?(sourceUrl=interceptionData.cachedRequestUrl,headers=Object.assign({},interceptionData.cachedRequestHeaders)):"cachedResponseHeaders"===headersSource?(sourceUrl=interceptionData.cachedResponseUrl,headers=Object.assign({},interceptionData.cachedResponseHeaders)):"origStateHeaders"===headersSource?(sourceUrl=interceptionData.origStateUrl,headers=Object.assign({},interceptionData.origStateHeaders)):"updatedRequestHeaders"===headersSource?(sourceUrl=interceptionData.updatedRequestUrl,headers=Object.assign({},interceptionData.updatedRequestHeaders)):"updatedResponseHeaders"===headersSource?(sourceUrl=interceptionData.updatedResponseUrl,headers=Object.assign({},interceptionData.updatedResponseHeaders)):"updatedStateHeaders"===headersSource?(sourceUrl=interceptionData.updatedStateUrl,headers=Object.assign({},interceptionData.updatedStateHeaders)):this.#procErrname("cfginvalid","actionApi","invalidHeadersSource"),headersSource.endsWith("Headers")&&(sourceUrl\|\|0===Array.from(Object.keys(headers)).length)){GetPara.hasSameOrigin(sourceUrl,reqUrl,headersSource)\|\|(headers={},this.#procErrname("cfginvalid","actionApi","differentOrigin"))}referer&&(headers.referer=referer);for(const headerCfg of headerCfgs){const name=this.#getCfgStrAttr(headerCfg,"name"),value=this.#getCfgStrAttr(headerCfg,"value");headers[name]=value}return headers}return null}catch(err){return loginfo(err),this.#procErrname("cfginvalid","actionApi","garhException"),null}}#setContentLengthHeaderIfNeeded(headers,requestData){return headers&&headers["Content-Length"]?(headers["Content-Length"]=String(requestData.length),!0):!(!headers\|\|!headers["content-length"])&&(headers["content-length"]=String(requestData.length),!0)}#getDataFromDataStr(dataStr,dataType){if("string"===dataType)return dataStr;if("number"===dataType)return Number(dataStr);if("boolean"===dataType)return!(!dataStr\|\|"false"===dataStr.toLowerCase()\|\|"0"===dataStr);if("null"===dataType)return null;if("undefined"===dataType)return;if("object"!==dataType)throw this.#procErrname("cfginvalid","actionApi","notObject"),new Error("unreachable");const data=(0,import_utils9.getJsonParseOfStr)(dataStr);if(!data\|\|"object"!=typeof data)throw this.#procErrname("cfginvalid","actionApi","jsonParseData"),new Error("unreachable");return data}#getRequestData(dataStr,dataType,requestCfgs,errName){try{const dataCfgs=requestCfgs.filter((requestCfg=>"data"===this.#getCfgStrAttr(requestCfg,"type")));if(dataCfgs.length>0&&"object"!==dataType)return this.#procErrname("cfginvalid","actionApi","invalidDataType"),"";if("object"!==dataType)return this.#getDataFromDataStr(dataStr,dataType);const data=this.#getDataFromDataStr(dataStr,dataType);for(const dataCfg of dataCfgs){const{type:type,subkeys:subkeys,value:value,datatype:datatype}=this.#transformStrAttrsInCfg(dataCfg);if("property"!==type\|\|!subkeys)throw this.#procErrname("cfginvalid","actionApi","invalidSubkeys"),new Error("unreachable");let subKeys=this.#getSubKeys(subkeys);const property=subKeys.pop(),parent=0===subKeys.length?data:this.#getMemberOfObject(data,subKeys);if(!parent\|\|"object"!=typeof parent)throw this.#procErrname("cfginvalid","actionApi","invalidParent"),new Error("unreachable");const propValue=this.#getDataFromDataStr(value,datatype);parent[property]=propValue}return data}catch(err){return loginfo(err),void this.#procErrname(errName,"actionApi","grdExeception")}}#setRequestData(requestOptions,context,formFlag,data){switch(context){case"browser":case"state":case"task":case"standalone":case"default":formFlag?requestOptions.form=data:requestOptions.data=data;break;case"fetch":if(formFlag){if(!data\|\|"object"!=typeof data)throw this.#procErrname("cfginvalid","actionApi","fetch"),new Error("unreachable");const formData=new FormData;for(const key of Object.keys(data))formData.set(key,data[key]);requestOptions.data=formData}else"string"==typeof data?requestOptions.body=data:requestOptions.data=data;break;default:(0,import_utils9.unreachable)(context)}return!0}async#getResponseOfApiRequest(context,timeout,reqUrl,requestOptions,errName){const response={statusCode:200,text:""};try{switch(context){case"browser":case"state":case"task":case"standalone":const apiContext=await this.#getApiContxt(context);if(!apiContext)throw new Error("##te null apiContext");timeout>0&&(requestOptions.timeout=timeout);const apiResponse=await apiContext.fetch(reqUrl,requestOptions);response.statusCode=apiResponse.status,response.text=apiResponse.text;break;case"default":timeout>0&&(requestOptions.timeout={request:timeout});const defaultResponse=await(0,import_got_scraping.gotScraping)(reqUrl,requestOptions);response.statusCode=defaultResponse.statusCode,response.text=defaultResponse.body;break;case"fetch":const fetchResponse=await fetch(reqUrl,requestOptions);response.statusCode=fetchResponse.status,response.text=await fetchResponse.text();break;default:(0,import_utils9.unreachable)(context)}return response}catch(err){return loginfo(err),this.#procErrname(errName,"actionApi","groarException"),response.statusCode=600,response.text=err.message,response}}#processStatusCodeOfApiResponse(actCfg,statusCode){let responseCfgs=this.#getCfgChildrenOf(actCfg,"response",!1);for(const responseCfg of responseCfgs){const codes=this.#getCfgStrAttr(responseCfg,"codes");if(!codes)throw this.#procErrname("cfginvalid","actionApi","invalideCodes"),new Error("unreachable");if(codes.split(",").map((code=>Number(code))).includes(statusCode)){const errName=this.#getCfgStrAttr(responseCfg,"errname");this.#procErrname(errName,"actionApi","statusCode")}}return!0}#validateApiResponseText(text,valErrName,pattern,flags){const re=this.#getRegExp(pattern,flags);if(re&&!text.match(re))throw this.#procErrname(valErrName,"actionApi","apiResponseText"),new Error("unreachable");return!0}#getCachedApiResponseText(origPath){try{const filename=this.#getAbsolutePath(origPath);return fs6.readFileSync(filename,"utf8")}catch(err){return""}}#getAbsolutePath(origPath){return"uuidv4"===origPath&&(origPath=`files/tid_${this.#templateId}_${uuid3.v4()}.txt`),origPath=path5.join(appConfig.cfg.baseFileDir,origPath)}#cacheApiResponse(requestMethod,requestUrl,requestData,responseData){const page=this.#taskContext.page,responseItem={pageUrl:page?page.url():"",requestUrl:requestUrl,requestMethod:requestMethod,requestData:requestData,responseData:responseData};return this.#interceptionData.responses.push(responseItem),!0}#saveApiResponse(text,origPath,errName){try{const filename=this.#getAbsolutePath(origPath);return fs6.writeFileSync(filename,text),!0}catch(err){return this.#procErrname(errName,"actionApi","sarExeception"),!1}}async#performApiAction(actCfg){let statusCode=200,text="",requestData="";const apiCfg=this.#transformStrAttrsInCfg(actCfg),{method:method,url:url,encodeuri:encodeUriFlag,session:sessionFlag,headerssource:headersSource,headers:headersStr,referer:referer,proxy:proxyFlag,timeout:timeoutStr,data:dataStr,datatype:dataType,form:formFlag,path:filePath,diskcache:diskCacheFlag,cache:cacheFlag,varname:varName,errname:errName,valerrname:valErrName,pattern:pattern,flags:flags}=apiCfg,context=apiCfg.context,timeout=timeoutStr?Number(timeoutStr):0,serverFlag=url.startsWith("/"),reqUrl=encodeUriFlag?encodeURI(url):url,requestCfgs=this.#getCfgChildrenOf(actCfg,"request",!1);if(!["GET","DELETE","POST","PUT"].includes(method.toUpperCase()))return this.#procErrname("cfginvalid","actionApi","invalidMethod"),"";if(serverFlag){if("function"==typeof GetPara.getOtherFunObj.getDataFromServerFun){const options=timeout>0?{timeout:timeout}:void 0;return await GetPara.getOtherFunObj.getDataFromServerFun(method,reqUrl,dataStr,options)}return""}{const requestOptions={method:method.toUpperCase()},proxy=this.#taskContext.proxy;proxy&&(proxyFlag&&proxy.proxyUrl&&(requestOptions.proxyUrl=proxy.proxyUrl),sessionFlag&&proxy.proxyUrl&&"default"===context&&(requestOptions.sessionToken=proxy.proxyUrl));const headers=this.#getApiRequestHeaders(reqUrl,headersSource,headersStr,referer,requestCfgs);if(headers&&(requestOptions.headers=headers),dataStr){const data=this.#getRequestData(dataStr,dataType,requestCfgs,errName);void 0!==data&&(requestData=JSON.stringify(data),this.#setRequestData(requestOptions,context,formFlag,data),this.#setContentLengthHeaderIfNeeded(headers,data))}let fromDiskFlag=!1;if(diskCacheFlag&&(text=this.#getCachedApiResponseText(filePath),text&&(fromDiskFlag=!0)),!text){const response=await this.#getResponseOfApiRequest(context,timeout,reqUrl,requestOptions,errName);statusCode=response.statusCode,text=response.text}return 200!==statusCode&&this.#processStatusCodeOfApiResponse(actCfg,statusCode),"ignore"!==valErrName&&pattern&&this.#validateApiResponseText(text,valErrName,pattern,flags),cacheFlag&&this.#cacheApiResponse(method,reqUrl,requestData,text),!fromDiskFlag&&filePath&&this.#saveApiResponse(text,filePath,errName),varName&&this.#setNormalVariable(varName,text),text}}async#performBreakAction(actCfg){this.#validateCfgType(actCfg),this.#taskContext.loopsStack.length>0?this.#taskContext.breakFlag=!0:logerr("##te Error: break cannot be called outside of loop")}#getLocOfChildElement(parentCfg){const elementCfg=this.#getRequiredCfgChildOf(parentCfg,"element");return this.#getCfgStrAttr(elementCfg,"loc")}#getLocsOfChildElements(parentCfg,minCnt=1){const elementCfgs=this.#getCfgChildrenOf(parentCfg,"element");return elementCfgs.length<minCnt?(this.#procErrname("cfginvalid","childElements","elementCnt"),[]):elementCfgs.map((elementCfg=>this.#getCfgStrAttr(elementCfg,"loc")))}async#processTextCaptcha(page,captchaCfg,proxy,tryTimes){const imageCfg=this.#getRequiredCfgChildOf(captchaCfg,"image"),imageLocs=this.#getLocsOfChildElements(imageCfg),commentCfg=this.#getOptionalCfgChildOf(captchaCfg,"comment"),commentLoc=commentCfg?this.#getLocOfChildElement(commentCfg):"",commentAttr=commentCfg?this.#getCfgStrAttr(commentCfg,"attr"):"",inputCfg=this.#getRequiredCfgChildOf(captchaCfg,"input"),inputLoc=this.#getLocOfChildElement(inputCfg),enter=this.#getCfgBoolAttr(inputCfg,"enter"),submitCfg=enter?null:this.#getOptionalCfgChildOf(captchaCfg,"submit"),submitLoc=enter\|\|!submitCfg?"":this.#getLocOfChildElement(submitCfg);if(!enter&&!submitCfg)return this.#procErrname("cfginvalid","actionCaptcha","invalidTextCfg"),!1;const checkCfg=this.#getRequiredCfgChildOf(captchaCfg,"check"),checkLoc=this.#getLocOfChildElement(checkCfg),checkAttr=this.#getCfgStrAttr(checkCfg,"attr"),failedStr=this.#getCfgStrAttr(checkCfg,"failedstr"),origCfg=this.#transformStrAttrsInCfg(captchaCfg),{lang:lang,math:math,maxlen:maxlen,minlen:minlen,question:question,space:space,texttype:texttype}=origCfg,options={type:"text",imageLocs:imageLocs,commentLoc:commentLoc,commentAttr:commentAttr,inputLoc:inputLoc,enter:enter,submitLoc:submitLoc,checkLoc:checkLoc,checkAttr:checkAttr,failedStr:failedStr,case:origCfg.case,lang:lang,math:math,maxLength:parseInt(maxlen),minLength:parseInt(minlen),question:question,space:space,textType:texttype};return await solveRecognitionCaptcha(page,options,proxy,tryTimes)}async#processCoordinateCaptcha(page,captchaCfg,proxy,tryTimes){const imageCfg=this.#getRequiredCfgChildOf(captchaCfg,"image"),imageLocs=this.#getLocsOfChildElements(imageCfg),commentCfg=this.#getOptionalCfgChildOf(captchaCfg,"comment"),commentLoc=commentCfg?this.#getLocOfChildElement(commentCfg):"",commentAttr=commentCfg?this.#getCfgStrAttr(commentCfg,"attr"):"";let submitLoc="";const submitCfg=this.#getOptionalCfgChildOf(captchaCfg,"submit");if(submitCfg&&(submitLoc=this.#getLocOfChildElement(submitCfg),!submitLoc))return this.#procErrname("cfginvalid","actionCaptcha","invalidCoordinateCfg"),!1;const checkCfg=this.#getRequiredCfgChildOf(captchaCfg,"check"),checkLoc=this.#getLocOfChildElement(checkCfg),checkAttr=this.#getCfgStrAttr(checkCfg,"attr"),failedStr=this.#getCfgStrAttr(checkCfg,"failedstr"),origCfg=this.#transformStrAttrsInCfg(captchaCfg),{lang:lang,maxclicks:maxclicks,minclicks:minclicks}=origCfg,options={type:"coordinate",imageLocs:imageLocs,commentLoc:commentLoc,commentAttr:commentAttr,submitLoc:submitLoc,checkLoc:checkLoc,checkAttr:checkAttr,failedStr:failedStr,lang:lang,maxClicks:parseInt(maxclicks),minClicks:parseInt(minclicks)};return await solveRecognitionCaptcha(page,options,proxy,tryTimes)}async#performCaptchaAction(actCfg){const page=this.#taskContext.page;if(!page)return this.#procErrname("cfginvalid","actionCaptcha","noPage"),!1;const tryTimes=parseInt(this.#getCfgStrAttr(actCfg,"try")),debug=this.#getCfgBoolAttr(actCfg,"debug"),captchaCfg=this.#getRequiredCfgChildOf(actCfg,"captcha"),captchaType=this.#getCfgStrAttr(captchaCfg,"type"),errName=this.#getCfgStrAttr(actCfg,"errname"),proxy=this.#taskContext.proxy;let result;switch(captchaType){case"amazon":case"funcaptcha":case"geetest":case"keycaptcha":case"mtcaptcha":case"turnstile":result=await solveTokenCaptcha(page,captchaType,proxy,{debug:debug});break;case"recaptcha":const minScore=Number(this.#getCfgStrAttr(captchaCfg,"minscore"));result=await solveTokenCaptcha(page,captchaType,proxy,{minScore:minScore,debug:debug});break;case"text":case"grid":case"slider":case"rotation":result=await this.#processTextCaptcha(page,captchaCfg,proxy,tryTimes);break;case"coordinate":result=await this.#processCoordinateCaptcha(page,captchaCfg,proxy,tryTimes);break;default:(0,import_utils9.unreachable)(captchaType)}return debug&&logwarn("##te captcha result: ",result),0!==result?.code&&this.#procErrname(errName,"actionCaptcha","cannotSolveCaptcha"),!0}async#performClickAction(actCfg){const page=this.#taskContext.page;if(!page)return this.#procErrname("cfginvalid","actionClick","noPage"),!1;this.#validateCfgType(actCfg);const actType=this.#getCfgStrAttr(actCfg,"type"),clickType=this.#getCfgStrAttr(actCfg,"clicktype"),tryTimes=parseInt(this.#getCfgStrAttr(actCfg,"try")),popupPageType=this.#getCfgStrAttr(actCfg,"popuppage"),navigateFlag=this.#getCfgBoolAttr(actCfg,"navigate"),errName=this.#getCfgStrAttr(actCfg,"errname"),eleCfgs=this.#getCfgChildrenOf(actCfg,"element",!0);"click"!==clickType&&"evaluate"!==clickType&&(logerr(`##task invalid clicktype ${clickType} in performClickAction`),this.#procErrname("cfginvalid","actionClick","invalidClickType"));const{elementSource:elementSource,elements:elements}=await this.#processElementCfgs(eleCfgs,tryTimes);if(this.#checkBrowserElementSource(elementSource,"click"),0===elements.length)return loginfo(`## te ---- no element to click, actCfg: ${JSON.stringify(actCfg).slice(0,200)}`),this.#procErrname(errName,"actionClick","noElements"),!1;const element=elements[0];try{logdbg(`##te click text: ${await element.innerText()}`)}catch(err){logdbg(err)}page.browserContext().browser().isHeadless()\|\|(logdbg(`##te pageId${page.id()} bringToFront in click`),await page.bringToFront());const downloadpath=this.#getCfgStrAttr(actCfg,"downloadpath"),baseDownloadDir=appConfig.cfg.baseDownloadDir,fullDownloadPath=path5.join(baseDownloadDir,downloadpath);if(downloadpath&&baseDownloadDir){if(!0===page.browserContext().browser().isHeadless()&&(logerr('##te please set browserHeadlesses "true" to download files by clicking'),this.#procErrname("cfginvalid","actionClick","invalidBrowserHeadlesses")),await(0,import_utils9.fsCheckOrCreateSubdir)(baseDownloadDir,downloadpath,"")){const method="Browser.setDownloadBehavior";try{loginfo(`##browser response of CDP ${method}: `,await page.sendCDPMessage(method,{behavior:"allow",downloadPath:fullDownloadPath}))}catch(err){err.message?.includes("does not support")?(logerr(err.message),this.#procErrname("cfginvalid","actionClick","invalidBrowserControllerTypes")):(logerr(err),logwarn(`##browser failed to send CDP message ${method}`))}}else this.#procErrname("other","actionClick","invalidDownloadDir")}const dataLoginCaptchaInfo=this.#getDataLoginCaptchaInfo(actCfg,actType);if(dataLoginCaptchaInfo&&dataLoginCaptchaInfo.popupFlag?await this.processNewPageLoginCaptcha(dataLoginCaptchaInfo,(async ele=>{await ele.click({clickType:clickType})}),[element]):await element.click({clickType:clickType}),"ignored"===popupPageType&&navigateFlag)try{await(0,import_utils9.sleep)(5e3)}catch(err){}dataLoginCaptchaInfo&&dataLoginCaptchaInfo.currentFlag&&await this.#processPageLoginCaptcha(page,dataLoginCaptchaInfo,"current");const waitMsStr=this.#getCfgStrAttr(actCfg,"wait");await this.#processWait(waitMsStr);const filename=this.#getCfgStrAttr(actCfg,"filename");let fullFilename="";if(fullDownloadPath&&filename){fullFilename=path5.join(fullDownloadPath,filename);let i=0;const maxLoops=300;for(i=0;i<maxLoops;i++){if(await(0,import_utils9.fileExist)(fullFilename))break;await(0,import_utils9.sleep)(1e3)}i>=maxLoops&&(logwarn(`##te download kId ${this.#taskId}: failed to download file ${fullFilename}`),this.#procErrname("other","actionClick","downloadTimeout"))}const varName=this.#getCfgStrAttr(actCfg,"pathvarname");return varName&&this.#setNormalVariable(varName,fullFilename),this.#addTaskCredits(actCfg),!0}async#performContinueAction(actCfg){return this.#validateCfgType(actCfg),this.#taskContext.loopsStack.length>0?(this.#taskContext.continueFlag=!0,!0):(logerr("##te Error: continue cannot be called outside of loop"),!1)}async#performExitAction(actCfg){this.#validateCfgType(actCfg);const errName=this.#getCfgStrAttr(actCfg,"errname");if(!errName)return!1;const credits=void 0!==this.#errObj[errName]?this.#errObj[errName]:-69;throw this.#taskContext.credits=credits,new Error(`##te action_throw with errname ${errName}`)}#getAbsoluteUrl(origUrl){try{const page=this.#taskContext.page,pageUrl=page?page.url():this.#paras.pageUrl?this.#paras.pageUrl:"";if(!pageUrl)return logwarn("Please execute page.goto or set variable pageUrl first"),origUrl;const url=new URL(origUrl,pageUrl);return url?url.href:origUrl}catch(err){return origUrl}}#getRegExp(pattern,flags=""){if(!pattern)return null;try{return flags?new RegExp(pattern,flags):new RegExp(pattern)}catch(err){return this.#procErrname("cfginvalid","regExp","invalidRegExp"),null}}async#processElecontentCfg(elecontentCfg,elements){let contentType="text";elecontentCfg&&(contentType=this.#getCfgStrAttr(elecontentCfg,"type"));let message="step00",content="",multielesFlag=!0,trimFlag=!1,joinStr="\n",onelineFlag=!1,topFlag=!1,absoluteFlag=!1,arr=[];try{if(elements.length>0){let firstElement=elements[0];switch(contentType){case"text":message="step11",elecontentCfg&&(multielesFlag=this.#getCfgBoolAttr(elecontentCfg,"multieles"),trimFlag=this.#getCfgBoolAttr(elecontentCfg,"trim"),joinStr=this.#getCfgStrAttr(elecontentCfg,"join"),onelineFlag=this.#getCfgBoolAttr(elecontentCfg,"line"),topFlag=this.#getCfgBoolAttr(elecontentCfg,"top")),message="step12";for(let element of elements){if(content=await element.innerText(topFlag),trimFlag&&(content=content.trim()),!multielesFlag)break;arr.push(content)}message="step13",multielesFlag&&(content="stringify"===joinStr?JSON.stringify(arr):arr.join(joinStr)),onelineFlag&&(content=content.replace(/[\s\r\n]+/g," ")),message="step14";break;case"attr":message="step21";const attrName=this.#getCfgStrAttr(elecontentCfg,"attrname"),attrName2=this.#getCfgStrAttr(elecontentCfg,"attrname2"),attrName3=this.#getCfgStrAttr(elecontentCfg,"attrname3"),attrName4=this.#getCfgStrAttr(elecontentCfg,"attrname4");trimFlag=this.#getCfgBoolAttr(elecontentCfg,"trim");const pattern=this.#getCfgStrAttr(elecontentCfg,"pattern"),boolAttrFlag=this.#getCfgBoolAttr(elecontentCfg,"boolattr");multielesFlag=this.#getCfgBoolAttr(elecontentCfg,"multieles"),joinStr=this.#getCfgStrAttr(elecontentCfg,"join"),onelineFlag=this.#getCfgBoolAttr(elecontentCfg,"line"),absoluteFlag=this.#getCfgBoolAttr(elecontentCfg,"absolute"),message="step22";for(let element of elements){if(boolAttrFlag)message="step22",content=await element.hasAttribute(attrName)?"1":"0";else{message="step23";const re=this.#getRegExp(pattern);for(const aName of[attrName,attrName2,attrName3,attrName4]){if(!aName)break;let attrVal=await element.attribute(aName);if(content="string"==typeof attrVal?attrVal:null===attrVal?"":String(attrVal),trimFlag&&(content=content.trim()),!re)break;if(content.match(re))break;content=""}absoluteFlag&&(content=this.#getAbsoluteUrl(content))}if(!multielesFlag)break;arr.push(content),message="step24"}multielesFlag&&(content="stringify"===joinStr?JSON.stringify(arr):arr.join(joinStr)),onelineFlag&&(content=content.replace(/[\s\r\n]+/g," ")),message="step25";break;case"innerhtml":content=await firstElement.innerHtml();break;case"length":content=String(elements.length);break;case"outerhtml":content=await firstElement.outerHtml();break;case"pictext":content="pictext";break;case"image":content="image";break;case"query":content="query";break;default:throw new Error(`##te Invalid content type: ${contentType}`)}}else"length"===contentType&&(content="0");return content}catch(err){throw loginfo(`##te processBrowserElecontentCfg: tId${this.#templateId} ecCfg: ${JSON.stringify(elecontentCfg).slice(0,200)}`),loginfo(`##te err: ${JSON.stringify(err)}`),loginfo(`##te contentType ${contentType}, content ${content}, onelineFlag ${onelineFlag}`),loginfo(`##te message ${message}`),loginfo(`##te $browserEles ${elements}, length ${elements.length}, e0 ${elements[0]}`),new Error(err)}}async#processTransformCfg(transformCfg,origContent){try{let retStr="";const defaulVal=this.#getCfgStrAttr(transformCfg,"defaultval"),usevarFlag=this.#getCfgBoolAttr(transformCfg,"usevar");let funCfgs=this.#getCfgChildrenOf(transformCfg,"fun",!0).map((cfg=>cfg[appConfig.XML_ATTR_KEY]));if(usevarFlag){const origFunCfgs=funCfgs;funCfgs=[];for(const origFunCfg of origFunCfgs){const funCfg=Object.assign({},origFunCfg);for(const key of Object.keys(funCfg))"string"==typeof funCfg[key]&&funCfg[key].includes("${")&&(funCfg[key]=this.#transTemplToStr(funCfg[key],this.#paras));funCfgs.push(funCfg)}}return retStr=await getTransformExpStr(origContent,funCfgs,defaulVal,this.#myfunScripts),retStr}catch(err){throw loginfo(`##te processTransformCfg: tId${this.#templateId} tCfg: ${JSON.stringify(transformCfg).slice(0,200)}`),loginfo(`##te err: ${JSON.stringify(err)}`),new Error(err)}}async#getSubtaskPage(popupFlag){if(popupFlag){if(!this.#taskContext.popupPage)throw new Error("No valid popup page");return this.#taskContext.popupPage}if(this.#taskContext.page){const page=await this.#taskContext.page.browserContext().getPage(!0);if(page)return page;throw new Error("No new page to perform sync subtask")}throw new Error("logic error in #getSubtaskPage")}async#freeSubtaskPage(page){return await page.free(),!0}#getSubtaskNetworkContext(page){const{proxy:proxy,browserApiContext:browserApiContext,standaloneApiContext:standaloneApiContext}=this.#taskContext;return{proxy:proxy,page:page,browserApiContext:browserApiContext,standaloneApiContext:standaloneApiContext}}async#processSyncSubtask(subtaskCfg){try{let content="";const templateId=parseInt(this.#getCfgStrAttr(subtaskCfg,"tid")),minLen=parseInt(this.#getCfgStrAttr(subtaskCfg,"minlen")),parasStr=this.#getCfgStrAttr(subtaskCfg,"parasstr");let errName=this.#getCfgStrAttr(subtaskCfg,"errname");const popuppageFlag=this.#getCfgBoolAttr(subtaskCfg,"popuppage");if(popuppageFlag&&!this.#taskContext.popupPage){const nopageErrName="ignore"===errName?"cfginvalid":errName;this.#procErrname(nopageErrName,"syncSubtask","noPopupPage")}if(parasStr.length>=minLen){const page=await this.#getSubtaskPage(popuppageFlag),{taskData:taskData}=await this.#performOneTask(templateId,parasStr,this.#getSubtaskNetworkContext(page),"memSync");await this.#freeSubtaskPage(page);const{credits:credits,execData:execData}=taskData;if(popuppageFlag&&(this.#taskContext.popupPage=null),logdbg(`##te memsyncsubtask t${templateId} p${parasStr} c${credits} data: ${JSON.stringify(execData)}`),credits>=0){if(execData&&1===Object.keys(execData).length&&Array.isArray(Object.values(execData)[0])){const tabData=Object.values(execData)[0];if(Array.isArray(tabData)&&1===tabData.length&&1===Object.keys(tabData[0]).length){const attrName=Object.keys(tabData[0])[0];content=tabData[0][attrName]}else content=JSON.stringify(execData)}else content=JSON.stringify(execData);let genCredits=calcGenCredits(credits),capCredits=calcCapCredits(credits);this.#addTaskCredits(null,!0,!0,genCredits,capCredits)}}else this.#procErrname(errName,"syncSubtask","failed");return content}catch(err){throw loginfo(`##te processSyncSubtask: tId${this.#templateId} pCfg: ${JSON.stringify(subtaskCfg).slice(0,200)}`),loginfo(`##te err: ${JSON.stringify(err)}`),new Error(err)}}#getGenHttpHeaderOptions(getCfg,origGetCfg){const genHeaderOptions={browserListQuery:getCfg.browserlist?getCfg.browerlist:"",browsers:["chrome"],operatingSystems:["windows"],devices:["desktop"],httpVersion:Number(getCfg.httpversion)>0?getCfg.httpversion:"2"},browserCfgs=this.#getCfgChildrenOf(origGetCfg,"httpbrowser");if(browserCfgs.length>0){const browserOptions=[];browserCfgs.forEach((browserCfg=>{const name=this.#getCfgStrAttr(browserCfg,"name");name\|\|this.#procErrname("cfginvalid","actGetHttpHeaders","invalidName");const browserOption={name:name},minVersion=parseInt(this.#getCfgStrAttr(browserCfg,"minversion"));minVersion>0&&(browserOption.minVersion=minVersion);const maxVersion=parseInt(this.#getCfgStrAttr(browserCfg,"maxversion"));maxVersion>0&&(browserOption.maxVersion=maxVersion);const httpVersion=this.#getCfgStrAttr(browserCfg,"httpversion");httpVersion&&Number(httpVersion)>0&&(browserOption.httpVersion=httpVersion),browserOptions.push(browserOption)})),genHeaderOptions.browsers=browserOptions}const osCfgs=this.#getCfgChildrenOf(origGetCfg,"httpos");if(osCfgs.length>0){const osOptions=[];osCfgs.forEach((osCfg=>{const osName=this.#getCfgStrAttr(osCfg,"os");osName\|\|this.#procErrname("cfginvalid","actGetHttpHeaders","invalidOsName"),osOptions.push(osName)})),genHeaderOptions.operatingSystems=osOptions}const deviceCfgs=this.#getCfgChildrenOf(origGetCfg,"httpdevice");if(deviceCfgs.length>0){const deviceOptions=[];deviceCfgs.forEach((deviceCfg=>{const deviceName=this.#getCfgStrAttr(deviceCfg,"device");deviceName\|\|this.#procErrname("cfginvalid","actGetHttpHeaders","invalidDeviceName"),deviceOptions.push(deviceName)})),genHeaderOptions.devices=deviceOptions}return genHeaderOptions}async#getEleOrTemplTransform(contentSrcType,parentCfg,objItem){try{let content="",selectedElements=[];if("element"===contentSrcType){const parentType=this.#getCfgStrAttr(parentCfg,"type"),eleCfgs=this.#getCfgChildrenOf(parentCfg,"element",!0),elecontentCfg=this.#getRequiredCfgChildOf(parentCfg,"elecontent"),tryStr=this.#getCfgStrAttr(parentCfg,"try",!0,!1),tryTimes=tryStr?parseInt(tryStr):1,{elements:elements}=await this.#processElementCfgs(eleCfgs,tryTimes);if(selectedElements=elements,0===selectedElements.length){if("length"===this.#getCfgStrAttr(elecontentCfg,"type"))content="0";else if("element"===parentType&&parentCfg[appConfig.XML_ATTR_KEY].colname){const errName=this.#getCfgStrAttr(parentCfg,"errname");this.#procErrname(errName,"columnElement","invalidColname")}}else content=await this.#processElecontentCfg(elecontentCfg,elements)}else if("templstr"===contentSrcType){const templstrCfg=this.#getRequiredCfgChildOf(parentCfg,"templstr");content=this.#getCfgStrAttr(templstrCfg,"templ")}else if("property"===contentSrcType){const subKeys=this.#getCfgStrAttr(parentCfg,"subkeys"),keys=this.#getSubKeys(subKeys),obj=this.#getMemberOfObject(objItem,keys);content=this.#getStringOfObj(obj)}else if("get"===contentSrcType){const origGetCfg=this.#getRequiredCfgChildOf(parentCfg,"get"),getCfg=this.#transformStrAttrsInCfg(origGetCfg);let page=this.#taskContext.page,targetType="page";const otherInfo={paras:this.#paras},type=this.#getCfgStrAttr(origGetCfg,"type");if("screenshot"===type){const eleCfg=this.#getOptionalCfgChildOf(origGetCfg,"element");if(eleCfg){const{elements:elements}=await this.#processElementCfgs([eleCfg],1);if(!(elements.length>0))throw new Error("##te no elements selected in get_screenshot");targetType="element",content=await GetPara.getParaInElement(getCfg,elements[0],otherInfo)}}else if("httpheaders"===type){const headerMethod=getCfg.headermethod;otherInfo.interceptionData=this.#interceptionData,"genNewByHttpOptions"===headerMethod&&(otherInfo.genHeaderOptions=this.#getGenHttpHeaderOptions(getCfg,origGetCfg));const newHttpHeaders=[];this.#getCfgChildrenOf(origGetCfg,"httpheader",!0).forEach((headerCfg=>{let name=this.#getCfgStrAttr(headerCfg,"name");if(name=name.toLowerCase().trim(),name){const source=this.#getCfgStrAttr(headerCfg,"source"),value=this.#getCfgStrAttr(headerCfg,"value"),deleteFlag=this.#getCfgBoolAttr(headerCfg,"delete");newHttpHeaders.push({name:name,source:source,value:value,deleteFlag:deleteFlag})}})),otherInfo.newHttpHeaders=newHttpHeaders}else"response"===type&&(otherInfo.interceptionData=this.#interceptionData);page&&"page"===targetType&&["content","cookies","mhtml","pdf","screenshot","title","window"].includes(getCfg.type)?content=await GetPara.getParaInPage(getCfg,page,otherInfo):["addedtaskparas","datetime","execedtaskparas","file","httpheaders","queuedcntwithparas","queuedtaskparas","random","response","special","sysparas"].includes(getCfg.type)?content=await GetPara.getPara(getCfg,otherInfo):["screenshot"].includes(getCfg.type)&&"element"===targetType\|\|this.#procErrname("cfginvalid","getPara","invalidType")}else if("subtask"===contentSrcType){const subtaskCfg=this.#getRequiredCfgChildOf(parentCfg,"subtask");loginfo(`##te to execsync subtaskCfg: ${JSON.stringify(subtaskCfg)}`),content=await this.#processSyncSubtask(subtaskCfg)}else{if("ocr"!==contentSrcType)throw new Error(`##te Invalid content source type: ${contentSrcType}`);{let ocrCfg=this.#getRequiredCfgChildOf(parentCfg,"ocrcfg");const eleCfgs=this.#getCfgChildrenOf(parentCfg,"element",!0),{elementSource:elementSource,elements:elements}=await this.#processElementCfgs(eleCfgs,1,!1);if(elements.length>0&&(this.#checkBrowserElementSource(elementSource,"ocr"),content=await this.__processBrowserOcrCfg(ocrCfg,elements),"number"==typeof content&&content<0))throw new Error(`##te Error with ocr content: ${content}`)}}const decodefontsvgCfg=this.#getOptionalCfgChildOf(parentCfg,"decodefontsvg"),decodefontttfCfg=this.#getOptionalCfgChildOf(parentCfg,"decodefontttf");(decodefontsvgCfg\|\|decodefontttfCfg)&&(content=await this.__decodeFontInContent(content,decodefontsvgCfg,decodefontttfCfg));const transformCfg=this.#getOptionalCfgChildOf(parentCfg,"transform");return transformCfg&&(content=await this.#processTransformCfg(transformCfg,content),null===content&&this.#procErrname("cfginvalid","decodeFont","invalidFontchars")),{content:content,elements:selectedElements}}catch(err){throw loginfo(`##te getEleOrTemplTransform: tId${this.#templateId} pCfg: ${JSON.stringify(parentCfg).slice(0,200)}`),loginfo(`##te err: ${JSON.stringify(err)}`),new Error(err)}}#getColumnStrValOfLastRec(tabName,columnName){let lastStrVal="";if(this.#taskContext.execData[tabName]&&this.#taskContext.execData[tabName].length>0){let recCnt=this.#taskContext.execData[tabName].length,lastVal=this.#taskContext.execData[tabName][recCnt-1][columnName];lastStrVal=String("boolean"==typeof lastVal?Number(lastVal):lastVal)}return lastStrVal}async#processColumnCfg(columnCfg,tableName,objItem){const valErrName=this.#getCfgStrAttr(columnCfg,"valerrname"),pattern=this.#getCfgStrAttr(columnCfg,"pattern");try{this.#validateCfgType(columnCfg);const columnType=this.#getCfgStrAttr(columnCfg,"type"),colName=this.#getCfgStrAttr(columnCfg,"colname");let{content:content}=await this.#getEleOrTemplTransform(columnType,columnCfg,objItem);if(0===content.length){this.#getCfgBoolAttr(columnCfg,"completed")&&(content=this.#getColumnStrValOfLastRec(tableName,colName))}if("ignore"!==valErrName&&pattern){const flags=this.#getCfgStrAttr(columnCfg,"flags"),re=this.#getRegExp(pattern,flags);re&&!content.match(re)&&this.#procErrname(valErrName,"columnContent","validate")}this.#getCfgBoolAttr(columnCfg,"setvar")&&this.#setNormalVariable(colName,content);const discardedFlag=this.#getCfgBoolAttr(columnCfg,"discarded"),retObj={};if(!discardedFlag){let retVal;const dataType=this.#getCfgStrAttr(columnCfg,"datatype");"number"===dataType?retVal=this.#getIntVal(content,"round"):"boolen"===dataType&&(retVal=this.#getBoolVal(content)),retObj[colName]="string"===dataType?content:retVal}return retObj}catch(err){throw this.#taskContext.credits>=0&&(loginfo(`##te processColumnCfg: tId${this.#templateId}, fCfg ${JSON.stringify(columnCfg).slice(0,100)}`),loginfo(`##te err: ${JSON.stringify(err)}`)),new Error(err)}}async#processExtractAction(actCfg){const tabName=this.#getCfgStrAttr(actCfg,"tabname");if("string"!=typeof tabName\|\|0===tabName.length)throw new Error(`##te Invalid table name: ${tabName}`);this.#taskContext.execData[tabName]\|\|(this.#taskContext.execData[tabName]=[]);const rec={},columnCfgs=this.#getCfgChildrenOf(actCfg,"column",!0);for(let columnCfg of columnCfgs){const colName=this.#getCfgStrAttr(columnCfg,"colname");if(!(this.#extractColumns.length>0)\|\|this.#extractColumns.includes(colName))try{let resObj=await this.#processColumnCfg(columnCfg,tabName);Object.assign(rec,resObj),colName.startsWith("cx")&&loginfo(`##dbg extract: ${colName}=${rec[colName]}`)}catch(err){throw this.#taskContext.credits>=0&&loginfo(`##te extract column tId${this.#templateId} fCfg: ${JSON.stringify(columnCfg).slice(0,200)}`),err}}return this.#taskContext.execData[tabName].push(rec),!0}#getMemberOfObject(obj,keys){let result=obj;if(0===keys.length)return result;for(const key of keys)if(result=result?.[key],void 0===result)return;return result}#getStringOfObj(obj){try{switch(typeof obj){case"string":return obj;case"number":case"bigint":return String(obj);case"boolean":return obj?"1":"0";case"object":return obj?JSON.stringify(obj):"";default:return""}}catch(err){return loginfo(err),""}}#checkIsNotNullObject(obj,errName){obj&&"object"==typeof obj\|\|this.#procErrname(errName,"actionExtractArray","notNormalObject")}#getSubKeys(subKeysStr){if(!subKeysStr.trim())return[];const keys=subKeysStr.trim().split(".");if(!keys.every((key=>!!key)))throw this.#procErrname("cfginvalid","actionExtractArray","invalidSubkeys"),new Error("unreachable");return keys}#getArrayItemsFromCachedResponses(reqUrl,subKeys,errName){const arrayItems=[],keys=this.#getSubKeys(subKeys);let responses=this.#interceptionData.responses;if(reqUrl){const re=new RegExp(reqUrl);responses=responses.filter((r=>r.requestUrl.match(re)))}for(const response of responses){const content=response.responseData,json=(0,import_utils9.getJsonParseOfStr)(content,null);this.#checkIsNotNullObject(json,errName);const objOrItems=this.#getMemberOfObject(json,keys);if(this.#checkIsNotNullObject(objOrItems,errName),Array.isArray(objOrItems)){if(!objOrItems.every((item=>item&&"object"==typeof item))){this.#procErrname(errName,"actionExtractArray","invalidResponseItems");continue}objOrItems.forEach((item=>arrayItems.push(item)))}else objOrItems&&"object"==typeof objOrItems&&arrayItems.push(objOrItems)}return arrayItems}#getArrayItemsFromString(list,subKeys,errName){let arrayItems=(0,import_utils9.getJsonParseOfStr)(list);if(this.#checkIsNotNullObject(arrayItems,errName),subKeys){const keys=this.#getSubKeys(subKeys);arrayItems=this.#getMemberOfObject(arrayItems,keys),this.#checkIsNotNullObject(arrayItems,errName)}return Array.isArray(arrayItems)?arrayItems.every((item=>item&&"object"==typeof item))?arrayItems:(this.#procErrname(errName,"actionExtractArray","invalidArrayItems"),[]):arrayItems&&"object"==typeof arrayItems?[arrayItems]:[]}async#processExtractArrayAction(actCfg){const{list:list,requrl:requrl,subkeys:subkeys,keys:keys,tabname:tabName,varname:varName,idxname:idxName,maxloops:maxloops,errname:errName}=this.#transformStrAttrsInCfg(actCfg);let arrayItems=list?this.#getArrayItemsFromString(list,subkeys,errName):this.#getArrayItemsFromCachedResponses(requrl,subkeys,errName);const maxLoops=parseInt(maxloops);maxLoops>0&&arrayItems.length>maxLoops&&(arrayItems=arrayItems.slice(0,maxLoops));let records=this.#taskContext.execData[tabName];if(records\|\|(records=[],this.#taskContext.execData[tabName]=records),0===arrayItems.length)return!0;const columnCfgs=this.#getCfgChildrenOf(actCfg,"column",!1),objKeys=keys?keys.split(","):0===columnCfgs.length?Array.from(Object.keys(arrayItems[0])):[];for(const[idx,objItem]of arrayItems.entries()){const rec={};varName&&this.#setNormalVariable(varName,JSON.stringify(objItem)),idxName&&this.#setNormalVariable(idxName,String(idx));for(const key of objKeys){const obj=this.#getMemberOfObject(objItem,[key]),colValue=this.#getStringOfObj(obj);rec[key]=colValue}for(let columnCfg of columnCfgs)try{const colName=this.#getCfgStrAttr(columnCfg,"colname");let resObj=await this.#processColumnCfg(columnCfg,tabName,objItem);Object.assign(rec,resObj),colName.startsWith("cx")&&loginfo(`##dbg extract: ${colName}=${rec[colName]}`)}catch(err){throw this.#taskContext.credits>=0&&loginfo(`##te extract column tId${this.#templateId} fCfg: ${JSON.stringify(columnCfg).slice(0,200)}`),err}records.push(rec)}return!0}async#getExtractScriptInData(maxLoops,htmlFlag,tabName,errName){try{const vars=Object.assign({},this.#paras,{authInfo:null,sysParas:null}),execData=this.#taskContext.execData,data={vars:vars,responses:this.#interceptionData.responses,tabName:tabName,maxLoops:maxLoops,errName:errName,execData:execData};if(htmlFlag&&this.#taskContext.page){const html=await this.#taskContext.page.content();data.html=html}return data}catch(err){return null}}#checkExtractScriptResult(result){if(!result\|\|"object"!=typeof result\|\|!result.execData\|\|"object"!=typeof result.execData)return!1;const{execData:execData,errName:errName}=result;for(const tabName of Object.keys(execData))if(!tabName.startsWith("dat_")\|\|20!==tabName.length)return!1;for(const records of Object.values(execData))if(!Array.isArray(records)\|\|!records.every((rec=>rec&&"object"==typeof rec)))return!1;return!!(void 0===errName\|\|"string"==typeof errName&&errName)}async#processExtractScriptAction(actCfg){const cfg=this.#transformStrAttrsInCfg(actCfg),{base64:base64Flag,html:htmlFlag,isolated:isolatedFlag,tabname:tabName,maxloops:maxLoopsStr,errname:errName}=cfg;let script=this.#getCfgTextContext(actCfg).trim();if(base64Flag&&(script=(0,import_utils9.decodeFromBase64)(script)),script.length<10\|\|!isolatedFlag)return this.#procErrname("cfginvalid","actionExtractScript","invalidScript"),!1;try{const maxLoops=parseInt(maxLoopsStr),inData=await this.#getExtractScriptInData(maxLoops,htmlFlag,tabName,errName);if(!inData)return this.#procErrname(errName,"actionExtractScript","invalidInData"),!1;const isolate=new import_isolated_vm2.default.Isolate,context=isolate.createContextSync(),externalCopy=new import_isolated_vm2.default.ExternalCopy(inData);context.global.setSync("inData",externalCopy.copyInto());const content=isolate.compileScriptSync(script).runSync(context);"string"==typeof content&&content\|\|logwarn("Please check the script of extract is correct.");const result=JSON.parse(content);if(!this.#checkExtractScriptResult(result))return this.#procErrname(errName,"actionExtractScript","invalidResult"),!1;result.errName&&("ignore"===result.errName?this.#procErrname(errName,"actionExtractScript","resultError1"):this.#procErrname(result.errName,"actionExtractScript","resultError2"));const execData=this.#taskContext.execData,newExecData=result.execData;for(const tabName2 of Object.keys(newExecData))execData[tabName2]=newExecData[tabName2];return!0}catch(err){return logwarn(err.message),this.#procErrname(errName,"actionExtractScript","pesaException"),!1}}async#isTableExtractable(vertical){let attrName="rowspan",eleCfg=this.#generateElementCfg(`[${attrName}]`);const{elements:rowElements}=await this.#processElementCfg(eleCfg);if(rowElements.length>0)for(const element of rowElements){const rowspan=await element.attribute(attrName);if(!rowspan&&parseInt(rowspan)>1)return!1}attrName="colspan",eleCfg=this.#generateElementCfg(`[${attrName}]`);const{elements:colElements}=await this.#processElementCfg(eleCfg);if(colElements.length>0)for(const element of colElements){const colspan=await element.attribute(attrName);if(!colspan&&parseInt(colspan)>1)return!1}return!0}#getTableHeader(origHeader,keyformat,colname,onlyDigitalCharFlag=!0){let headerStr=onlyDigitalCharFlag&&"camel"===keyformat?origHeader.replaceAll(/[^\w\|\s]/g,"").replaceAll(/_/g," "):origHeader;if(colname&&"string"==typeof colname)return colname;if("header"===keyformat)return headerStr;{const words=headerStr.split(/\s+/);let header=words[0].toLowerCase();for(const word of words.slice(1))header=`${header}${word[0].toUpperCase()}${word.slice(1).toLowerCase()}`;return header}}async#extractHorizontalTable(keyformat,colnames,tabname,maxLoops,errName){const headers=[],records=[];let hasThead=!1,spaceIdx=1,trLocPrefix="",recordStartIdx=1,eleCfg=this.#generateElementCfg("./thead/tr");const{elements:theadTrElements}=await this.#processElementCfg(eleCfg);if(theadTrElements.length>1)return this.#procErrname(errName,"actionExtractTable","manyTheadTrs"),!0;if(1===theadTrElements.length){hasThead=!0,eleCfg=this.#generateElementCfg("./thead/tr/th");const{elements:theadTrThElements}=await this.#processElementCfg(eleCfg);if(0===theadTrThElements.length)return this.#procErrname(errName,"actionExtractTable","noTheadTrTh"),!0;for(const[idx,th]of theadTrThElements.entries()){const origHeader=(await th.textContent()).trim(),header=this.#getTableHeader(origHeader\|\|"space"+spaceIdx++,keyformat,colnames[idx]);headers.push(header)}}trLocPrefix="./tr",eleCfg=this.#generateElementCfg(trLocPrefix);let{elements:trElements}=await this.#processElementCfg(eleCfg);if(0===trElements.length){trLocPrefix="./tbody/tr",eleCfg=this.#generateElementCfg(trLocPrefix);let{elements:tbodyTrElements}=await this.#processElementCfg(eleCfg);trElements=tbodyTrElements}if(trElements.length<=1)return this.#procErrname(errName,"actionExtractTable","noRecord"),!0;if(!hasThead){eleCfg=this.#generateElementCfg(`${trLocPrefix}[${recordStartIdx}]/`);let{elements:thElements}=await this.#processElementCfg(eleCfg);if(0===thElements.length)return this.#procErrname(errName,"actionExtractTable","noTr"),!0;for(const[idx,th]of thElements.entries()){const origHeader=(await th.textContent()).trim(),header=this.#getTableHeader(origHeader\|\|"space"+spaceIdx++,keyformat,colnames[idx]);headers.push(header)}recordStartIdx++,trElements=trElements.slice(1)}const headersCnt=headers.length;for(const recordIdx of trElements.keys()){eleCfg=this.#generateElementCfg(`${trLocPrefix}[${recordStartIdx+recordIdx}]/`);let{elements:colElements}=await this.#processElementCfg(eleCfg);if(colElements.length!==headersCnt)return this.#procErrname(errName,"actionExtractTable","differentColumns"),!0;const record={};for(const colIdx of colElements.keys()){if(maxLoops>0&&colIdx>maxLoops)break;const val=await colElements[colIdx].textContent();record[headers[colIdx]]=val}records.push(record)}const origData=this.#taskContext.execData[tabname];return Array.isArray(origData)?records.forEach((record=>origData.push(record))):this.#taskContext.execData[tabname]=records,!0}async#extractVerticalTable(keyformat,colnames,tabname,maxLoops,errName){const records=[];let recordsCnt=0,trLocPrefix="",spaceIdx=1;trLocPrefix="./tr";let eleCfg=this.#generateElementCfg(trLocPrefix),{elements:trElements}=await this.#processElementCfg(eleCfg);if(0===trElements.length){trLocPrefix="./tbody/tr",eleCfg=this.#generateElementCfg(trLocPrefix);let{elements:tbodyTrElements}=await this.#processElementCfg(eleCfg);trElements=tbodyTrElements}if(0===trElements.length)return this.#procErrname(errName,"actionExtractTable","noTrElments"),!0;for(const trIdx of trElements.keys()){let thCfg=this.#generateElementCfg(`${trLocPrefix}[${trIdx+1}]/th`),{elements:thElements}=await this.#processElementCfg(thCfg);if(1!==thElements.length)return this.#procErrname(errName,"actionExtractTable","noThElements"),!0;const origHeader=(await thElements[0].textContent()).trim(),header=this.#getTableHeader(origHeader\|\|"space"+spaceIdx++,keyformat,colnames[trIdx]);let tdCfg=this.#generateElementCfg(`${trLocPrefix}[${trIdx+1}]/td`),{elements:tdElements}=await this.#processElementCfg(tdCfg);if(0===tdElements.length)return this.#procErrname(errName,"actionExtractTable","noTdElements"),!0;if(0===trIdx){for(const tdIdx of tdElements.keys()){if(maxLoops>0&&tdIdx>maxLoops)break;const val=await tdElements[tdIdx].textContent(),record={};record[header]=val,records.push(record)}recordsCnt=records.length}else{if(tdElements.length<recordsCnt)return this.#procErrname(errName,"actionExtractTable","differentVertCols"),!0;for(const tdIdx of tdElements.keys()){if(tdIdx>=recordsCnt)break;const val=await tdElements[tdIdx].textContent();records[tdIdx][header]=val}}}const origData=this.#taskContext.execData[tabname];return Array.isArray(origData)?records.forEach((record=>origData.push(record))):this.#taskContext.execData[tabname]=records,!0}async#processExtractTableAction(actCfg){const{orientation:orientation,check:check,tabname:tabname,keyformat:keyformat,colnames:colnamesStr,maxloops:maxloops,errname:errName}=this.#transformStrAttrsInCfg(actCfg),colnames=colnamesStr?colnamesStr.split(","):[],eleCfgs=this.#getCfgChildrenOf(actCfg,"element",!0),maxLoops=parseInt(maxloops);if(maxLoops<0)return!1;if(0===eleCfgs.length)return this.#procErrname("logicerror","actionExtractTable","noEleCfg"),!1;const{elementSource:elementSource,elements:tables}=await this.#processElementCfg(eleCfgs[0]);if(0===tables.length)return this.#procErrname(errName,"actionExtractTable","noTables"),!0;const element=tables[0];if("browser"===elementSource?this.#taskContext.browserElesStack.push(element):"cheerio"===elementSource?this.#taskContext.cheerioElesStack.push(element):(0,import_utils9.unreachable)(elementSource),this.#taskContext.loopsStack.push(element),check){if(!await this.#isTableExtractable("vertical"===orientation))return this.#procErrname("cfginvalid","actionExtractTable","notExtractable"),!1}return"vertical"===orientation?await this.#extractVerticalTable(keyformat,colnames,tabname,maxLoops,errName):await this.#extractHorizontalTable(keyformat,colnames,tabname,maxLoops,errName),this.#taskContext.loopsStack.pop(),"browser"===elementSource?this.#taskContext.browserElesStack.pop():"cheerio"===elementSource?this.#taskContext.cheerioElesStack.pop():(0,import_utils9.unreachable)(elementSource),!0}async#performExtractAction(actCfg){this.#validateCfgType(actCfg);let subType=this.#getCfgStrAttr(actCfg,"subtype",!1,!1);if(subType)if("array"===subType)await this.#processExtractArrayAction(actCfg);else if("script"===subType)await this.#processExtractScriptAction(actCfg);else{if("table"!==subType)return this.#procErrname("cfginvalid","actionExtract","logicError"),!1;await this.#processExtractTableAction(actCfg)}else await this.#processExtractAction(actCfg);return!0}#getHttpHeadersObj(headers){try{const headersObj=JSON.parse(headers);return headersObj&&"object"==typeof headersObj?headersObj:null}catch(err){this.#procErrname("cfginvalid","actionGoto","invalidHeaders")}}async#performGotoAction(actCfg){const page=this.#taskContext.page;if(!page)return this.#procErrname("cfginvalid","actionGogo","noPage"),!1;this.#taskContext.browserElesStack.length>0&&this.#procErrname("cfginvalid","actionGoto","inLoopOfBrowserElements");let url="";try{this.#validateCfgType(actCfg);const actType=this.#getCfgStrAttr(actCfg,"type");url=this.#getCfgStrAttr(actCfg,"url");const waitMsStr=this.#getCfgStrAttr(actCfg,"wait"),reuseFlag=this.#getCfgBoolAttr(actCfg,"reuse"),encodeFlag=this.#getCfgBoolAttr(actCfg,"encodeuri"),headers=this.#getCfgStrAttr(actCfg,"headers"),referer=this.#getCfgStrAttr(actCfg,"referer");if(encodeFlag&&(url=encodeURI(url)),!this.#validateUrl(url))throw new Error(`##te Invalid url: ${url} when trying to open this page`);if(loginfo(`##te goto url ${url}`),reuseFlag){if(page.url().includes(url))return await this.#processWait(waitMsStr),this.#addTaskCredits(actCfg),page}if(this.#lastUrl=url,headers.trim()){const httpHeadersObj=this.#getHttpHeadersObj(headers.trim());httpHeadersObj&&await page.setExtraHTTPHeaders(httpHeadersObj)}this.#canResetPage(page,actType);const dataLoginCaptchaInfo=this.#getDataLoginCaptchaInfo(actCfg,actType);return dataLoginCaptchaInfo&&dataLoginCaptchaInfo.popupFlag?await this.processNewPageLoginCaptcha(dataLoginCaptchaInfo,(async(p,u,r)=>{r?await p.goto(u,{waitUntil:"load",timeout:0,referer:r}):await p.goto(u,{waitUntil:"load",timeout:0})}),[page,url,referer]):await page.goto(url),dataLoginCaptchaInfo&&dataLoginCaptchaInfo.currentFlag&&await this.#processPageLoginCaptcha(page,dataLoginCaptchaInfo,"current"),await this.#processWait(waitMsStr),this.#addTaskCredits(actCfg),page}catch(err){if(loginfo(`##te kId${this.#taskId} goto: ${url}`),-46!==this.#taskContext.credits&&(logerr(`##te kId${this.#taskId} error credits ${this.#taskContext.credits} in performGotoAction`),logerr(err)),this.#taskContext.credits>=0){const message=err.message?err.message:"";if(message.includes("net::ERR_INTERNET_DISCONNECTED"))this.#setFailedCredits(-71);else if(message.includes("net::ERR_NAME_NOT_RESOLVED")){await(0,import_utils9.checkDns)(appConfig.CHECK_DNS_DOMAIN)?this.#setFailedCredits(-69):this.#setFailedCredits(-71)}else message.includes("net::ERR_CONNECTION_TIMED_OUT")?this.#setFailedCredits(-72):message.includes("net::ERR_CONNECTION_REFUSED")\|\|message.includes("net::ERR_CONNECTION_RESET")?this.#setFailedCredits(-98):message.includes("net::ERR_PROXY_CONNECTION_FAILED ")\|\|message.includes("net::ERR_EMPTY_RESPONSE")\|\|message.includes("net::ERR_TIMED_OUT")\|\|message.includes("net::ERR_TUNNEL_CONNECTION_FAILED")?this.#setFailedCredits(-86):message.includes("net::ERR_INVALID_AUTH_CREDENTIALS")?this.#setFailedCredits(-88):(message.includes("Navigation timeout"),this.#setFailedCredits(-85))}throw new Error(err)}}async#handleHover(element,tryTimes=10){let error=null;for(let i=0;i<tryTimes;i++)try{return await element.hover(),!0}catch(err){loginfo(err),await(0,import_utils9.sleep)(2e3),error=err}throw error}async#performHoverAction(actCfg){const page=this.#taskContext.page;if(!page)return this.#procErrname("cfginvalid","actionHover","noPage"),!1;this.#validateCfgType(actCfg);const tryTimes=parseInt(this.#getCfgStrAttr(actCfg,"try")),errName=this.#getCfgStrAttr(actCfg,"errname");try{const eleCfgs=this.#getCfgChildrenOf(actCfg,"element",!0),{elementSource:elementSource,elements:elements}=await this.#processElementCfgs(eleCfgs,tryTimes);if(this.#checkBrowserElementSource(elementSource,"hover"),0===elements.length)return this.#procErrname(errName,"actionHover","noElements"),page;this.#isHeadless()&&await page.bringToFront(),logdbg(`##te pageIdx${this.#getPageInfo().pageIdx} hover`),await this.#handleHover(elements[0]);const waitMsStr=this.#getCfgStrAttr(actCfg,"wait");return await this.#processWait(waitMsStr),this.#addTaskCredits(actCfg),page}catch(err){return loginfo(err),this.#procErrname("other","actionHover","phaException"),page}}async#processConditionCfg(conditionCfg){this.#validateCfgType(conditionCfg);const conditionType=this.#getCfgStrAttr(conditionCfg,"type");let conditionResult=!1;if(["element","templstr"].includes(conditionType)){let content="";content=(await this.#getEleOrTemplTransform(conditionType,conditionCfg)).content,conditionResult=this.#getBoolVal(content)}else{if("else"!==conditionType)throw new Error("##te Invalid type(condition)");conditionResult=!0}if(conditionResult){const subactions=this.#getCfgChildrenOf(conditionCfg,"action");await this.#performActions(subactions)}return conditionResult}async#performIfelseAction(actCfg){this.#validateCfgType(actCfg);const conditionCfgs=this.#getCfgChildrenOf(actCfg,"condition");for(let conditionCfg of conditionCfgs)if(await this.#processConditionCfg(conditionCfg))break;return!0}async#performInputAction(actCfg){if(!this.#taskContext.page)return this.#procErrname("cfginvalid","actionInput","noPage"),!1;this.#validateCfgType(actCfg);const content=this.#getCfgStrAttr(actCfg,"content"),replace=this.#getCfgBoolAttr(actCfg,"replace"),enter=this.#getCfgBoolAttr(actCfg,"enter"),tryTimes=parseInt(this.#getCfgStrAttr(actCfg,"try")),errName=this.#getCfgStrAttr(actCfg,"errname"),eleCfgs=this.#getCfgChildrenOf(actCfg,"element",!0);try{const{elementSource:elementSource,elements:elements}=await this.#processElementCfgs(eleCfgs,tryTimes);if(this.#checkBrowserElementSource(elementSource,"input"),0===elements.length)return this.#procErrname(errName,"actionInput","noElements"),!1;await elements[0].input(content,{replace:replace,enter:enter});const waitMsStr=this.#getCfgStrAttr(actCfg,"wait");return await this.#processWait(waitMsStr),this.#addTaskCredits(actCfg),!0}catch(err){return logerr(err),!1}}#transformStrAttrsInCfg(origCfg,varFlag=!0){const obj=Object.assign({},origCfg[appConfig.XML_ATTR_KEY]);if(!obj\|\|"object"!=typeof obj)throw new Error("Invalid origCfg in transformStrAttrsInCfg");if(!varFlag)return obj;for(const key of Object.keys(obj))"string"==typeof obj[key]&&obj[key].includes("${")&&(obj[key]=this.#transTemplToStr(obj[key],this.#paras));return obj}#getRequestInterceptionOptions(requestCfgs){const options=[];for(const requestCfg of requestCfgs){const{type:type,method:method,url:url,resourcetype:resourcetype,postdata:postdata}=requestCfg,requestMatch={};if(method&&"all"!==method&&(requestMatch.methods=[method.toUpperCase()]),url&&(requestMatch.url=new RegExp(url)),resourcetype&&(requestMatch.resourceTypes=[resourcetype]),postdata&&(requestMatch.postData=new RegExp(postdata)),"abort"===type){const option={requestMatch:requestMatch,action:"abort"};options.push(option)}else{if("fufill"!==type)throw new Error(`Invalid action type ${type}`);{const{fulfill:fulfill}=requestCfg,option={requestMatch:requestMatch,action:"fulfill",fulfill:fulfill};options.push(option)}}}return options}#getResponseInterceptionOptions(responseCfgs){const options=[];for(const responseCfg of responseCfgs){const{type:type,method:method,url:url,resourcetype:resourcetype,postdata:postdata,minsize:minsize,maxsize:maxsize}=responseCfg,requestMatch={};method&&"all"!==method&&(requestMatch.methods=[method.toUpperCase()]),url&&(requestMatch.url=new RegExp(url)),resourcetype&&(requestMatch.resourceTypes=[resourcetype]),postdata&&(requestMatch.postData=new RegExp(postdata));const responseMatch={},minSize=parseInt(minsize);minSize>0&&(responseMatch.minLength=minSize);const maxSize=parseInt(maxsize);if(maxSize>0&&(responseMatch.maxLength=maxSize),"cache"===type){const{tabname:tabname,contenttype:contenttype,responses:responseFlag,requestheaders:requestHeadersFlag,responseheaders:responseHeadersFlag}=responseCfg,option={requestMatch:requestMatch,responseMatch:responseMatch,handler:responseCacheCb,handlerOptions:{execData:this.#taskContext.execData,contentType:contenttype,tabname:tabname,responseFlag:responseFlag,requestHeadersFlag:requestHeadersFlag,responseHeadersFlag:responseHeadersFlag,interceptionData:this.#interceptionData}};options.push(option)}else{if("save"!==type)throw new Error(`Invalid response type ${type}`);{const{basedir:basedir,encoding:encoding,extname:extname,hashmethod:hashmethod,hostnameparts:hostnameparts,params:params,pathtype:pathtype,tabname:tabname,validextnames:validextnames}=responseCfg,option={requestMatch:requestMatch,responseMatch:responseMatch,handler:responseSaveCb,handlerOptions:{execData:this.#taskContext.execData,baseDir:basedir,pathType:pathtype,hashMethod:hashmethod,hostnameParts:parseInt(hostnameparts),extname:extname,validExtnames:validextnames,params:params,encoding:encoding,tabname:tabname}};options.push(option)}}}return options}async#performInterceptAction(actCfg){const page=this.#taskContext.page;if(!page)return this.#procErrname("cfginvalid","actionIntercept","noPage"),!1;this.#validateCfgType(actCfg);const subType=this.#getCfgStrAttr(actCfg,"subtype");let requestCfgs=this.#getCfgChildrenOf(actCfg,"request"),responseCfgs=this.#getCfgChildrenOf(actCfg,"response");try{if(requestCfgs=requestCfgs.map((requestCfg=>this.#transformStrAttrsInCfg(requestCfg))),responseCfgs=responseCfgs.map((responseCfg=>this.#transformStrAttrsInCfg(responseCfg))),"set"===subType){if(requestCfgs.length<=0&&responseCfgs.length<=0&&this.#procErrname("cfginvalid","actionIntercept","noCfgs"),requestCfgs.length>0){const options=this.#getRequestInterceptionOptions(requestCfgs);await page.setRequestInterception(options)}if(responseCfgs.length>0){const options=this.#getResponseInterceptionOptions(responseCfgs);await page.setResponseInterception(options)}}else"clear"===subType&&(await page.clearRequestInterceptions(),await page.clearResponseInterceptions());return!0}catch(err){return logerr(err),!1}}async#performLoopforAction(actCfg){this.#validateCfgType(actCfg);try{const errName=this.#getCfgStrAttr(actCfg,"errname");let roundType=this.#getCfgStrAttr(actCfg,"roundtype"),fromInt=this.#getIntVal(this.#getCfgStrAttr(actCfg,"from"),roundType),toInt=this.#getIntVal(this.#getCfgStrAttr(actCfg,"to"),roundType),stepInt=this.#getIntVal(this.#getCfgStrAttr(actCfg,"step"),roundType),varName=this.#getCfgStrAttr(actCfg,"varname"),maxLoops=this.#getIntVal(this.#getCfgStrAttr(actCfg,"maxloops"),roundType);if(maxLoops<0)return!1;let loops=0;for(let i=fromInt;i<=toInt&&!(maxLoops&&loops++>=maxLoops);i+=stepInt){this.#taskContext.loopsStack.push(i),this.#setNormalVariable(varName,String(i));const subactions=this.#getCfgChildrenOf(actCfg,"action");if(await this.#performActions(subactions),this.#taskContext.loopsStack.pop(),this.#taskContext.continueFlag&&(this.#taskContext.continueFlag=!1),this.#taskContext.breakFlag){this.#taskContext.breakFlag=!1;break}}return loops<1&&this.#procErrname(errName,"actionLoopfor","zeroLoops"),!0}catch(err){throw err}}async#processElementCfg(eleCfg){try{const selector=this.#getCfgStrAttr(eleCfg,"loc").trim(),elementSource=this.#getElementSource(eleCfg),absoluteFlag=this.#getCfgBoolAttr(eleCfg,"absolute"),iframeOptions=this.#getCfgChildrenOf(eleCfg,"iframe",!1).map((cfg=>this.#transformStrAttrsInCfg(cfg))).map((cfg=>{const{srcprefix:src,loc:selector2}=cfg;return{src:src,selector:selector2}})),browserPage=this.#taskContext.page,retObj={elementSource:elementSource,elements:[]};if("browser"===elementSource){if(!browserPage)throw new Error("Cannot get elements from null browser page");if(0===this.#taskContext.browserElesStack.length)retObj.elements=await browserPage.findElements(selector,iframeOptions);else{const parent=this.#taskContext.browserElesStack.slice(-1)[0];retObj.elements=await parent.findElements(selector,iframeOptions,absoluteFlag)}return retObj}if("cheerio"===elementSource){if(iframeOptions.length>0)throw new Error("Iframe config is invalid in cheerio source");let cheerioPage=this.#taskContext.cheerioPage;if(!cheerioPage){if(!browserPage)throw new Error("Please set CheerioPage first, from which to select elememnts");const content=await browserPage.content();content.length>10&&(this.#setCheerioPage(content,!0),cheerioPage=this.#taskContext.cheerioPage)}if(!cheerioPage)throw new Error("Cannot get elements from null cheerioPage");if(0===this.#taskContext.cheerioElesStack.length)retObj.elements=await cheerioPage.findElements(selector);else{const parent=this.#taskContext.cheerioElesStack.slice(-1)[0];retObj.elements=await parent.findElements(selector,iframeOptions,absoluteFlag)}return retObj}(0,import_utils9.unreachable)(elementSource)}catch(err){throw loginfo(`##te processElementCfg: tId${this.#templateId} eleCfg: ${JSON.stringify(eleCfg).slice(0,200)}`),new Error(err)}}async#processElementCfgs(eleCfgs,tryTimes=1,firstSelectedFlag=!0){try{if(0===eleCfgs.length)throw new Error("No element tags");const retObj={elementSource:"browser",elements:[]};for(let i=0;i<tryTimes;i++){let lastElementSource="";const selectedElements=[];for(const eleCfg of eleCfgs){const{elementSource:elementSource,elements:elements}=await this.#processElementCfg(eleCfg);if(lastElementSource&&lastElementSource!=elementSource)throw new Error("Element sources of all element tags must be the same");if(lastElementSource=elementSource,elements.length>0&&(elements.forEach((e=>selectedElements.push(e))),firstSelectedFlag))break}if(lastElementSource&&(retObj.elementSource=lastElementSource),selectedElements.length>0)return retObj.elements=selectedElements,retObj;i>0&&(loginfo(`##tmp ${i} try failed in getEleOrTemplTransform`),await(0,import_utils9.sleep)(1e3))}return retObj}catch(err){throw loginfo(`##te processElementCfgs: eleCfgs: ${JSON.stringify(eleCfgs).slice(0,200)}`),new Error(err)}}async#performLoopinelesAction(actCfg){if(!this.#taskContext.page)return this.#procErrname("cfginvalid","actionLoopineles","noPage"),!1;this.#validateCfgType(actCfg);const varName=this.#getCfgStrAttr(actCfg,"varname"),positionFlag=varName.startsWith("position"),errName=this.#getCfgStrAttr(actCfg,"errname"),eleCfgs=this.#getCfgChildrenOf(actCfg,"element",!0);let maxLoops=this.#getIntVal(this.#getCfgStrAttr(actCfg,"maxloops"),"round"),start=this.#getIntVal(this.#getCfgStrAttr(actCfg,"start"),"round"),end=this.#getIntVal(this.#getCfgStrAttr(actCfg,"end"),"round"),step=this.#getIntVal(this.#getCfgStrAttr(actCfg,"step"),"round");if(step<1&&(step=1),maxLoops<0)return!1;const{elementSource:elementSource,elements:elements}=await this.#processElementCfgs(eleCfgs,1),selectedElesCnt=elements.length;if(0===selectedElesCnt)return this.#procErrname(errName,"actionLoopineles","noElements"),!0;start<=0&&(start+=selectedElesCnt),end<=0?end+=selectedElesCnt:end>selectedElesCnt&&(end=selectedElesCnt);let processedNum=0;for(let i=start-1;i<end&&!(maxLoops&&processedNum>=maxLoops);i+=step){varName&&this.#setNormalVariable(varName,String(positionFlag?i+1:i));const element=elements[i];"browser"===elementSource?this.#taskContext.browserElesStack.push(element):"cheerio"===elementSource?this.#taskContext.cheerioElesStack.push(element):(0,import_utils9.unreachable)(elementSource),this.#taskContext.loopsStack.push(element);const subactions=this.#getCfgChildrenOf(actCfg,"action");if(await this.#performActions(subactions),this.#taskContext.loopsStack.pop(),"browser"===elementSource?this.#taskContext.browserElesStack.pop():"cheerio"===elementSource?this.#taskContext.cheerioElesStack.pop():(0,import_utils9.unreachable)(elementSource),processedNum++,this.#taskContext.continueFlag&&(this.#taskContext.continueFlag=!1),this.#taskContext.breakFlag){this.#taskContext.breakFlag=!1;break}}return!0}async#performLoopinstrAction(actCfg){this.#validateCfgType(actCfg);let regStr=this.#getCfgStrAttr(actCfg,"split");const trimFlag=this.#getCfgBoolAttr(actCfg,"trim"),errName=this.#getCfgStrAttr(actCfg,"errname");let regExp;try{regExp=new RegExp(regStr)}catch(error){throw new Error(`##te Invalid regexp: ${regExp}`)}const loopItems=trimFlag?this.#getCfgStrAttr(actCfg,"list").trim().split(regExp).map((item=>item.trim())).filter((item=>item.length>0)):this.#getCfgStrAttr(actCfg,"list").split(regExp),varName=this.#getCfgStrAttr(actCfg,"varname");let maxLoops=this.#getIntVal(this.#getCfgStrAttr(actCfg,"maxloops"),"round");if(maxLoops<0)return!1;let loops=0;for(const item of loopItems){if(maxLoops&&loops++>=maxLoops)break;this.#taskContext.loopsStack.push(item),this.#setNormalVariable(varName,item);const subactions=this.#getCfgChildrenOf(actCfg,"action");if(await this.#performActions(subactions),this.#taskContext.loopsStack.pop(),this.#taskContext.continueFlag&&(this.#taskContext.continueFlag=!1),this.#taskContext.breakFlag){this.#taskContext.breakFlag=!1;break}}return loops<1&&this.#procErrname(errName,"actionLoopinstr","zeroLoops"),!0}async#performLoopdowhileAction(actCfg){this.#validateCfgType(actCfg);const subType=this.#getCfgStrAttr(actCfg,"subtype"),isWhileFlag=this.#getCfgBoolAttr(actCfg,"iswhile"),varName=this.#getCfgStrAttr(actCfg,"varname"),clickFlag="element"===subType&&this.#getCfgBoolAttr(actCfg,"click"),genFlag=(this.#getCfgBoolAttr(actCfg,"navigate"),"element"===subType&&this.#getCfgBoolAttr(actCfg,"gen")),capFlag="element"===subType&&this.#getCfgBoolAttr(actCfg,"cap"),errName=clickFlag?this.#getCfgStrAttr(actCfg,"errname"):"ignore",waitMsStr=clickFlag?this.#getCfgStrAttr(actCfg,"wait"):"-1";let maxLoops=this.#getIntVal(this.#getCfgStrAttr(actCfg,"maxloops"),"round");if(maxLoops<0)return!1;let{content:content,elements:elements}=await this.#getEleOrTemplTransform(subType,actCfg),loops=0;for(;0===loops&&!isWhileFlag\|\|this.#getBoolVal(content);){varName&&this.#setNormalVariable(varName,String(loops)),"element"!==subType\|\|!clickFlag\|\|this.#skipActions.includes("click")\|\|this.#skipActions.includes("operate")\|\|(elements.length<=0?this.#procErrname(errName,"actionLoopdowhile","noElements"):(await elements[0].click(),this.#addTaskCredits(actCfg,genFlag,capFlag))),this.#taskContext.loopsStack.push(loops);const subactions=this.#getCfgChildrenOf(actCfg,"action");if(await this.#performActions(subactions),this.#taskContext.loopsStack.pop(),this.#taskContext.continueFlag&&(this.#taskContext.continueFlag=!1),this.#taskContext.breakFlag){this.#taskContext.breakFlag=!1;break}if(await this.#processWait(waitMsStr),loops++,maxLoops&&loops>=maxLoops)break;let retObj=await this.#getEleOrTemplTransform(subType,actCfg);content=retObj.content,elements=retObj.elements}return!0}async#performGetstatedataInMisc(miscCfg){const cookiesSource=this.#getCfgStrAttr(miscCfg,"cookies"),localStorageSource=this.#getCfgStrAttr(miscCfg,"localstorage"),headersSource=this.#getCfgStrAttr(miscCfg,"headers"),userDataSource=this.#getCfgStrAttr(miscCfg,"userdata"),{cookies:cookies,localStorage:localStorage,userData:userData}=this.#taskContext.origStateData,stateData={cookies:[],localStorage:[],headers:{},userData:{}};let apiStateData;if("orig"===cookiesSource)stateData.cookies=cookies;else if("new"===cookiesSource)if(this.#taskContext.page)stateData.cookies=await this.#taskContext.page.cookies();else{if(!this.#taskContext.standaloneApiContext)throw new Error(`##templ ${this.#templateId} has no page when getting stateData of page`);apiStateData\|\|(apiStateData=await this.#taskContext.standaloneApiContext.stateData()),stateData.cookies=apiStateData.cookies}if("orig"===localStorageSource)stateData.localStorage=localStorage;else if("new"===localStorageSource)if(this.#taskContext.page)stateData.localStorage=await this.#taskContext.page.localStroage();else{if(!this.#taskContext.standaloneApiContext)throw new Error(`##templ ${this.#templateId} has no page when getting stateData of page`);apiStateData\|\|(apiStateData=await this.#taskContext.standaloneApiContext.stateData()),stateData.localStorage=apiStateData.localStorage}const interceptionData=this.#interceptionData;interceptionData[headersSource]&&(stateData.headers=Object.assign({},interceptionData[headersSource])),"orig"===userDataSource?stateData.userData=userData:"new"===userDataSource&&(stateData.userData=Object.assign({},this.#paras.userData)),this.#taskContext.newStateData=stateData}async _performGetstatedataInMisc(miscCfg){return await this.#performGetstatedataInMisc(miscCfg)}async#performScrollAction(actCfg){const page=this.#taskContext.page;if(!page)return this.#procErrname("cfginvalid","actionScroll","noPage"),!1;this.#validateCfgType(actCfg);const subType=this.#getCfgStrAttr(actCfg,"subtype");let height=["by","to"].includes(subType)?parseInt(this.#getCfgStrAttr(actCfg,"height")):0;const unitType=["by","to"].includes(subType)?this.#getCfgStrAttr(actCfg,"unit"):"none";let scrollHeight=0,unitStr="";"bodypct"===unitType?(unitStr=await page.windowMember(["document","body","scrollHeight"]),scrollHeight=parseInt(unitStr)):"screenpct"===unitType?unitStr=await page.windowMember(["screen","height"]):"windowpct"===unitType&&(unitStr=await page.windowMember(["innerHeight"]));const unitInt=unitStr?parseInt(unitStr):1;if(["bodypct","screenpct","windowpct"].includes(unitType)){if("number"!=typeof unitInt\|\|unitInt<=0)throw new Error("##te invalid unitInt in performScrollAction");height=Math.floor(heightunitInt/100)}let maxTimes="by"===subType?parseInt(this.#getCfgStrAttr(actCfg,"maxtimes")):0;const interval="by"===subType?parseInt(this.#getCfgStrAttr(actCfg,"interval")):0;if("by"===subType){if(0===maxTimes){if(scrollHeight<=0){const heightStr=await page.windowMember(["document","body","scrollHeight"]);scrollHeight=parseInt(heightStr)}maxTimes=Math.ceil(scrollHeight/height-1)}else maxTimes>200?maxTimes=200:maxTimes<0&&(maxTimes=10);for(let i=0;i<maxTimes;i++){await page.scrollBy(0,height),await(0,import_utils9.sleep)(interval);const eleCfgs=this.#getCfgChildrenOf(actCfg,"element");if(eleCfgs&&eleCfgs.length>0){const{elementSource:elementSource,elements:elements}=await this.#processElementCfgs(eleCfgs,1);if(this.#checkBrowserElementSource(elementSource,"scrollBy"),elements.length>0)break}}}else if("to"===subType)await page.scrollTo(0,height);else if("intoview"===subType){const eleCfgs=this.#getCfgChildrenOf(actCfg,"element",!0),{elementSource:elementSource,elements:elements}=await this.#processElementCfgs(eleCfgs,1);if(this.#checkBrowserElementSource(elementSource,"scrollIntoView"),elements.length<=0){const errName=this.#getCfgStrAttr(actCfg,"errname");return this.#procErrname(errName,"actionScrollIntoview","noElements"),!1}await elements[0].scrollIntoView()}return this.#addTaskCredits(actCfg),!0}async#selectOneOption(eleCfgs,selectType,selectVal,errName,tryTimes=1){if(eleCfgs.length<=0)throw new Error("##te No element cfg in select action");const{elementSource:elementSource,elements:elements}=await this.#processElementCfgs(eleCfgs,tryTimes);if(this.#checkBrowserElementSource(elementSource,"select"),elements.length<=0)return this.#procErrname(errName,"actionSelect","noElements"),!1;const options={type:"value"},items=selectVal.split(",");return"index"===selectType?(options.type,options.indexes=items.map((item=>parseInt(item)))):"label"===selectType?(options.type,options.labels=items):"value"===selectType?(options.type,options.values=items):(0,import_utils9.unreachable)(selectType),await elements[0].select(options),!0}async#performSelectAction(actCfg){const page=this.#taskContext.page;if(!page)return this.#procErrname("cfginvalid","actionSelect","noPage"),!1;page.title,this.#validateCfgType(actCfg);const selectType=this.#getCfgStrAttr(actCfg,"selecttype"),selectVal=this.#getCfgStrAttr(actCfg,"selectval"),tryTimes=parseInt(this.#getCfgStrAttr(actCfg,"try")),errName=this.#getCfgStrAttr(actCfg,"errname"),eleCfgs=this.#getCfgChildrenOf(actCfg,"element",!0);await this.#selectOneOption(eleCfgs,selectType,selectVal,errName,tryTimes);const waitMsStr=this.#getCfgStrAttr(actCfg,"wait");return await this.#processWait(waitMsStr),this.#addTaskCredits(actCfg),!0}async#performSetvarAction(actCfg){this.#validateCfgType(actCfg);const subType=this.#getCfgStrAttr(actCfg,"subtype"),capSubtype=subType.slice(0,1).toUpperCase()+subType.slice(1),varName=this.#getCfgStrAttr(actCfg,"varname"),errName=this.#getCfgStrAttr(actCfg,"errname"),valErrName=this.#getCfgStrAttr(actCfg,"valerrname"),pattern=this.#getCfgStrAttr(actCfg,"pattern"),defaultVal=this.#getCfgStrAttr(actCfg,"defaultval");let targetFilename=this.#getCfgStrAttr(actCfg,"path"),content="";try{if(["element","get","templstr","subtask","ocr"].includes(subType)){content=(await this.#getEleOrTemplTransform(subType,actCfg)).content}else if("dbquery"===subType)content=await this.__processDbquery(actCfg,defaultVal);else{if("file"!==subType)throw new Error(`##te Invalid subtupe ${subType} in setvar action`);{const fileCfg=this.#getRequiredCfgChildOf(actCfg,"file");let sourceFilename=this.#getCfgStrAttr(fileCfg,"path");const encoding=this.#getCfgStrAttr(fileCfg,"encoding"),proxyFlag=this.#getCfgBoolAttr(actCfg,"proxy"),lcFilename=sourceFilename.toLocaleLowerCase();if(sourceFilename.includes("..")&&this.#procErrname("cfginvalid",`actionSetvar${capSubtype}`,"invalidSourceFilename"),lcFilename.startsWith("http://")\|\|lcFilename.startsWith("https://")){const proxyUrl=proxyFlag&&this.#taskContext.proxy?this.#taskContext.proxy.proxyUrl:"";content=(await lsdDownload(sourceFilename,proxyUrl)).toString(encoding)}else path5.isAbsolute(sourceFilename)?appConfig.cfg.inputFileDirPart&&!path5.dirname(sourceFilename).split(path5.sep).includes(appConfig.cfg.inputFileDirPart)&&"ignore"===errName&&this.#procErrname("normal",`actionSetvar${capSubtype}`,"normalExit"):sourceFilename=path5.join(appConfig.cfg.baseFileDir,sourceFilename),content=fs6.readFileSync(sourceFilename,{encoding:encoding})}}if("ignore"!==valErrName&&pattern){const flags=this.#getCfgStrAttr(actCfg,"flags"),re=this.#getRegExp(pattern,flags);re&&!content.match(re)&&this.#procErrname(valErrName,`actionSetvar${capSubtype}`,"validate")}return!content&&defaultVal&&(content=defaultVal),varName&&this.#setNormalVariable(varName,content),targetFilename&&(targetFilename=this.#getAbsolutePath(targetFilename),fs6.writeFileSync(targetFilename,content)),!0}catch(err){return loginfo(err),content="",this.#procErrname(errName,`actionSetvar${capSubtype}`,"psaException"),!1}}async#performSubtaskAction(actCfg){this.#validateCfgType(actCfg);const subtasksStr=this.#getCfgStrAttr(actCfg,"subtasks"),sameAsParentFlag=this.#getCfgBoolAttr(actCfg,"sameasparent");if(subtasksStr&&subtasksStr.startsWith("[{"))try{const subtasks=JSON.parse(subtasksStr);if(Array.isArray(subtasks)&&subtasks.every((s=>s.tid>0&&"string"==typeof s.parasstr)))for(const st of subtasks){if(this.#taskContext.subtasks.find((item=>item.tid===st.tid&&item.parasstr===st.parasstr))\|\|st.minlen>0&&st.parasstr.length<st.minlen)continue;const subtask={tid:st.tid,parasstr:st.parasstr};st.idx>0&&(subtask.idx=st.idx),sameAsParentFlag&&(subtask.sapFlag=!0),this.#taskContext.subtasks.push(subtask)}else subtasks.length>=1&&("string"==typeof subtasks[0].parasstr&&"number"==typeof subtasks[0].tid\|\|logerr(`##te invalid subtask: ${JSON.stringify(subtasks[0])} in performSubtaskAction`))}catch(err){logerr(err)}const subtaskCfgs=this.#getCfgChildrenOf(actCfg,"subtask");for(const subtaskCfg of subtaskCfgs){const templateId=parseInt(this.#getCfgStrAttr(subtaskCfg,"tid")),minLen=parseInt(this.#getCfgStrAttr(subtaskCfg,"minlen")),parasStr=this.#getCfgStrAttr(subtaskCfg,"parasstr"),idxStr=this.#getCfgStrAttr(subtaskCfg,"idx").trim(),idx=idxStr.length>0&&idxStr.match(/^[0-9]+$/)?parseInt(idxStr):0,errName=this.#getCfgStrAttr(subtaskCfg,"errname"),popuppageFlag=this.#getCfgBoolAttr(subtaskCfg,"popuppage");if(popuppageFlag&&!this.#taskContext.popupPage){const nopageErrName="ignore"===errName?"cfginvalid":errName;this.#procErrname(nopageErrName,"actionSubtask","noPopupPage")}if(parasStr.length>=minLen)if(popuppageFlag){const page=await this.#getSubtaskPage(popuppageFlag),subtaskResult=await this.#performOneTask(templateId,parasStr,this.#getSubtaskNetworkContext(page),"indSync");await this.#freeSubtaskPage(page),this.#taskContext.popupPage=null;const{credits:credits,execData:execData}=subtaskResult.taskData;credits<-1?this.#procErrname(errName,"actionSubtask","SyncIndepFailed"):(this.#taskContext.subtaskResults.push(subtaskResult),logdbg(`##te indsyncsubtask t${templateId} p${parasStr} c${credits} data: ${JSON.stringify(execData)}`))}else{if(!this.#taskContext.subtasks.find((item=>item.tid===templateId&&item.parasstr===parasStr))){const subtask={tid:templateId,parasstr:parasStr};idx>0&&(subtask.idx=idx),sameAsParentFlag&&(subtask.sapFlag=!0),this.#taskContext.subtasks.push(subtask)}}}return!0}async#performWaitAction(actCfg){this.#validateCfgType(actCfg);const subType=this.#getCfgStrAttr(actCfg,"subtype"),errName=this.#getCfgStrAttr(actCfg,"errname"),waitMsStr=this.#getCfgStrAttr(actCfg,"wait");if("sleep"===subType){const waitTime1InMs=parseInt(this.#getCfgStrAttr(actCfg,"minms")),waitTime2InMs=parseInt(this.#getCfgStrAttr(actCfg,"maxms")),waitTime=(0,import_utils9.getRandomInt)(waitTime1InMs,waitTime2InMs);return await(0,import_utils9.sleep)(waitTime),await this.#processWait(waitMsStr),!0}const page=this.#taskContext.page;if("element"===subType){if(!page)return this.#procErrname("cfginvalid","actionWaitElement","noPage"),!1;const eleCfgs=this.#getCfgChildrenOf(actCfg,"element",!0);eleCfgs.length<=0&&this.#procErrname("cfginvalid","actionWaitElement","noElements");const selector=this.#getCfgStrAttr(eleCfgs[0],"loc"),timeout=parseInt(this.#getCfgStrAttr(actCfg,"timeout")),state=this.#getCfgStrAttr(actCfg,"state");try{await page.waitForElement(selector,{timeout:timeout,state:state})}catch(err){logerr(err),this.#procErrname(errName,"actionWaitElement","pwaException")}}else if("navigation"===subType){if(!page)return this.#procErrname("cfginvalid","actionWaitNavigation","noPage"),!1;const timeout=parseInt(this.#getCfgStrAttr(actCfg,"timeout")),waitUntil=this.#getCfgStrAttr(actCfg,"waituntil"),urlStr=this.#getCfgStrAttr(actCfg,"url");waitUntil\|\|this.#procErrname("cfginvalid","actionWaitNavigation","invlidWaitUntil");try{if(urlStr){const url=new RegExp(urlStr);await page.waitForNavigation({url:url,timeout:timeout,waitUntil:waitUntil})}else await page.waitForNavigation({timeout:timeout,waitUntil:waitUntil})}catch(err){logerr(err),this.#procErrname(errName,"actionWaitNavigation","pwaException")}}else this.#procErrname("cfginvalid","actionWait","invalidSubtype");return await this.#processWait(waitMsStr),!0}async#performActions(actCfgs){if(!Array.isArray(actCfgs))throw new Error("##te Ivalid action configs(actions)");for(const actCfg of actCfgs)try{const actType=this.#getCfgStrAttr(actCfg,"type");if(this.#skipActions.includes(actType)\|\|this.#skipActions.includes("operate")&&appConfig.OPERATE_ACTIONS.includes(actType))continue;if(await this.#actionPerformers[actType].call(this,actCfg),this.#taskContext.continueFlag\|\|this.#taskContext.breakFlag)break}catch(err){throw this.#taskContext.credits>=0&&(loginfo(`##te tId${this.#templateId}, aCfg: ${JSON.stringify(actCfg).slice(0,200)}`),loginfo(`##te err.stack: ${err.stack}`)),err}}async#execTaskActions(){try{const page=this.#taskContext.page,captchaTypes=this.#captchaOptions.captchaTypes;page&&captchaTypes.length>0&&await injectScripts(page,captchaTypes);const subactions=this.#getCfgChildrenOf(this.#actionConfigs,"action");await this.#performActions(subactions),this.emit(appConfig.EVENT_TYPES.taskComplete,{code:0,message:"ok",data:null})}catch(err){this.emit(appConfig.EVENT_TYPES.taskComplete,{code:1001,message:"failed",data:err})}}#getNormalVariables(){const variables={};for(const[key,value]of Object.entries(this.#paras))"string"==typeof key&&"string"==typeof value&&(variables[key]=value);return variables}#addListeners(){const page=this.#taskContext.page;if(page){page.on(appConfig.EVENT_TYPES.lsdPageClose,(()=>{this.emit(appConfig.EVENT_TYPES.taskComplete,{code:1005,message:"pageClosed",data:null})}));const maxExecutionDuration=this.#template.maxExecutionDuration;maxExecutionDuration>0&&setTimeout((()=>this.emit(appConfig.EVENT_TYPES.taskComplete,{code:1009,message:"timeout",data:null})),1e3maxExecutionDuration),page.pageInfo().relatedId>0&&page.on(appConfig.EVENT_TYPES.lsdPagePopup,(async eventData=>{loginfo("##tmp LsdPage received popup event: ",eventData),this.emit(appConfig.EVENT_TYPES.lsdScraperPopup,{code:0,message:"ok",data:eventData})}))}}#removeListeners(){const page=this.#taskContext.page;page&&(page.removeAllListeners(appConfig.EVENT_TYPES.lsdPagePopup),page.removeAllListeners(appConfig.EVENT_TYPES.lsdPageClose)),this.removeAllListeners(appConfig.EVENT_TYPES.taskComplete)}async run(){const taskData={templateId:this.#templateId,parasStr:this.#parasStr,credits:0,execData:{},subtasks:[]},page=this.#taskContext.page;let subtaskDatas;loginfo(page?`##tmp -- start to run tId${this.#templateId} kId${this.#taskId} page ${page.id()}`:`##tmp -- start to run tId${this.#templateId} kId${this.#taskId} no page`);try{this.#addListeners(),this.#execTaskActions();const[response]=await(0,import_node_events2.once)(this,appConfig.EVENT_TYPES.taskComplete);if(0===response.code);else if(1001===response.code){if(response.data instanceof Error)throw response.data;logerr(response.data)}else 1005===response.code?this.#procErrname("other","task","pageClosed"):1009===response.code&&this.#procErrname("tetimeout","task","teTimeout");page&&this.#lastUrl!==page.url()&&loginfo(`##te kId${this.#taskId} endUrl: ${page.url()}`);const subtaskCnt=this.#taskContext.subtasks.length;subtaskCnt>0&&loginfo(`##te kId${this.#taskId} has ${subtaskCnt} subtasks`);for(let tablename of Object.keys(this.#taskContext.execData)){const recordsCnt=this.#taskContext.execData[tablename].length;loginfo(`##te kId${this.#taskId} tablename ${tablename} has ${recordsCnt}`)}return this.#removeListeners(),loginfo(`##tmp teSucceeded kId${this.#taskId} tId${this.#templateId}`),taskData.execData=this.#taskContext.execData,taskData.subtasks=this.#taskContext.subtasks,taskData.credits=this.#taskContext.credits,this.#taskContext.subtaskResults.length>0&&(subtaskDatas=this.#taskContext.subtaskResults.map((tr=>tr.taskData))),await this.__whenTaskEndFun(),taskData.credits<-30&&logwarn(`##te taskId ${this.#taskId} credits ${taskData.credits}`),{taskData:taskData,subtaskDatas:subtaskDatas,newStateData:this.#taskContext.newStateData?this.#taskContext.newStateData:void 0}}catch(err){taskData.execData=this.#taskContext.execData,taskData.subtasks=this.#taskContext.subtasks,taskData.credits=this.#taskContext.credits,this.#taskContext.subtaskResults.length>0&&(subtaskDatas=this.#taskContext.subtaskResults.map((tr=>tr.taskData)));const newStateData=this.#taskContext.newStateData?this.#taskContext.newStateData:void 0,stack="string"==typeof err?.stack?err.stack:"unkown",misc={taskId:this.#taskId,message:"unknown",stack:stack,variables:this.#getNormalVariables()};if(this.#removeListeners(),await this.__whenTaskEndFun(),this.#taskContext.credits>=0)logerr(`##te Exeception in ScraperTask.run: kId ${this.#taskId}`),logerr(err);else{if(-1===this.#taskContext.credits)return{taskData:taskData,subtaskDatas:subtaskDatas,newStateData:newStateData,misc:misc};if(this.#taskContext.credits>=-30)return{taskData:taskData,subtaskDatas:subtaskDatas,newStateData:newStateData,misc:misc};if(2===this.#taskId&&"check"===this.#parasStr&&page){logdbg(`##cap loginCheckFailed capabilityId ${0} credits: ${this.#taskContext.credits}`)}else 11===this.#taskId\|\|-69===this.#taskContext.credits\|\|logwarn(`##te taskExec run execption: templateId ${this.#templateId}, paras ${this.#parasStr}, credits ${this.#taskContext.credits}`)}return logwarn(`##tmp taskExec run execption: templateId ${this.#templateId}, paras ${this.#parasStr}, credits ${this.#taskContext.credits}`),taskData.credits=this.#taskContext.credits>=0?-99:this.#taskContext.credits,{taskData:taskData,subtaskDatas:subtaskDatas,newStateData:newStateData,misc:misc}}}async __performMiscAction(actCfg){this.#validateCfgType(actCfg);const miscCfgs=this.#getCfgChildrenOf(actCfg,"misc");if(miscCfgs.length<=0)return logerr(`##templ ${this.#templateId} invalid template ${this.#templateId} in performMiscAction`),!1;for(const miscCfg of miscCfgs){const type=this.#getCfgStrAttr(miscCfg,"type");switch(type){case"delsyncdbdata":case"extractdata":this.#procErrname("cfginvlid","notSupported",type);break;case"getstatedata":await this.#performGetstatedataInMisc(miscCfg);break;default:(0,import_utils9.unreachable)(type)}}return!0}async __processDbquery(actCfg,defaultVal){return(actCfg\|\|defaultVal)&&this.#procErrname("cfginvlid","notSupported","dbquery"),""}async __processBrowserOcrCfg(ocrCfg,$browserEles){return(ocrCfg\|\|$browserEles)&&this.#procErrname("cfginvlid","notSupported","ocr"),""}async __decodeFontInContent(content,decodefontsvgCfg,decodefontttfCfg){return(content\|\|decodefontsvgCfg\|\|decodefontttfCfg)&&this.#procErrname("cfginvlid","notSupported","decodeFont"),""}async __whenTaskEndFun(){return!0}},path6=__toESM(require("path"),1),fsp2=__toESM(require("fs/promises"),1),import_utils10=require("@letsscrapedata/utils");async function moveDataFiles(){try{const exportsDir=appConfig.cfg.exportsDir;if(!exportsDir\|\|!(0,import_utils10.isDirectory)(exportsDir))return!1;const filenames=await(0,import_utils10.filesInDir)(exportsDir,"general","dat_");if(0===filenames.length)return!0;const dateNum=(0,import_utils10.getLocalDateNumber)(),oldDataDir=path6.join(exportsDir,String(dateNum));await(0,import_utils10.fsCheckOrCreateDir)(oldDataDir,!0,!0);for(const filename of filenames){const oldFilename=path6.join(exportsDir,filename),newFilename=path6.join(oldDataDir,filename);await fsp2.rename(oldFilename,newFilename)}return!0}catch(err){return logerr(err),!1}}async function writeDataIntoOneFile(filename,records,columnSeperator){if(records.length<=0)return!1;const dataFileFormat=appConfig.cfg.dataFileFormat;if("jsonl"!==dataFileFormat&&!await(0,import_utils10.isFile)(filename)){const keys=Array.from(Object.keys(records[0]));await fsp2.writeFile(filename,keys.join(columnSeperator).concat("\n"))}const content="jsonl"===dataFileFormat?records.map((r=>JSON.stringify(r))).join("\n"):records.map((r=>Object.values(r).join(columnSeperator))).join("\n");return await fsp2.appendFile(filename,content.concat("\n")),!0}async function saveExecDataOfTask(execData){try{const{exportsDir:exportsDir,dataFileFormat:dataFileFormat,columnSeperator:columnSeperator}=appConfig.cfg;if(!(exportsDir&&(0,import_utils10.isDirectory)(exportsDir)&&dataFileFormat&&columnSeperator))return!1;for(const[datatableName,records]of Object.entries(execData)){const filename=path6.join(exportsDir,`${datatableName}.${dataFileFormat}`);await writeDataIntoOneFile(filename,records,columnSeperator)}return!0}catch(err){return logerr(err),!1}}async function performOneTask(templateId,parasStr,taskNetworkContext,taskType="indAsync",xmlStr="",taskId=0){try{const parsedTemplateExt=await TemplateManagerInScraper.getTemplateConfig(templateId,xmlStr),{template:template,actionConfigs:actionConfigs,fontsConfig:fontsConfig,myfunScripts:myfunScripts,paraCfgs:paraCfgs,attrsInXml:attrsInXml,captchaTypes:captchaTypes}=parsedTemplateExt,templateConfig={template:template,actionConfigs:actionConfigs,fontsConfig:fontsConfig,myfunScripts:myfunScripts},{skipActions:skipActions="",extractColumns:extractColumns="",defaultParasStr:defaultParasStr}=attrsInXml\|\|{},taskParasStr=parasStr\|\|(appConfig.cfg.useParasstrInXmlIfNeeded&&defaultParasStr?defaultParasStr:""),inParas=TaskParser.getInParas(taskParasStr,paraCfgs),taskOptions=1===taskId&&attrsInXml?{skipActions:skipActions?skipActions.split(","):[],extractColumns:extractColumns?extractColumns.split(","):[]}:{};taskOptions.captchaOptions={captchaTypes:captchaTypes};const scraper2=new TaskScraper(templateConfig,inParas,taskNetworkContext,taskOptions),taskResult=await scraper2.run();if("indSync"===taskType\|\|"indAsync"===taskType)if(taskResult.taskData.credits>=-1)loginfo(`Task are completed: templateId ${templateId}, parasStr ${parasStr}`),await saveExecDataOfTask(taskResult.taskData.execData);else if("indSync"===taskType){const execData={dat_0000000000000001:[{c01:String(templateId),c02:parasStr,c03:String(taskResult.taskData.credits),c04:(0,import_utils11.getLocalDateString)()}]};await saveExecDataOfTask(execData)}return taskResult}catch(err){const misc={taskId:0,message:err.message,stack:err.stack,variables:{}};if("indSync"===taskType){const execData={dat_0000000000000001:[{c01:String(templateId),c02:parasStr,c03:"-1",c04:(0,import_utils11.getLocalDateString)()}]};await saveExecDataOfTask(execData)}return{taskData:{templateId:templateId,parasStr:parasStr,credits:0,execData:{},subtasks:[]},subtaskDatas:[],misc:misc}}}appConfig.performOneTask=performOneTask;var path8=__toESM(require("path"),1),import_utils12=require("@letsscrapedata/utils"),import_controller4=require("@letsscrapedata/controller"),BrowserManagerInScraper=class{#config;#controller;#browsers;#browserContextInfos;#getMaxConcurrency(templateId){const cfg=this.#config.templateParas.find((tmc=>tmc.templateId===templateId));return cfg?.maxConncurrency?cfg.maxConncurrency:this.#config.maxConcurrency}async#addBrowserContext(browserContext,proxy){const standaloneApiContext=await import_controller4.controller.newApiContext({proxy:proxy}),browserApiContext="playwright"===browserContext.browser().browserControllerType()?browserContext.apiContext():null;this.#browserContextInfos.push({proxy:proxy,browserContext:browserContext,browserApiContext:browserApiContext,standaloneApiContext:standaloneApiContext,templateMap:new Map})}constructor(){const{useStealthPlugin:useStealthPlugin,browserControllerType:browserControllerType,browserType:browserType,lsdLaunchOptions:lsdLaunchOptions,lsdConnectOptions:lsdConnectOptions,maxConcurrency:maxConcurrency,templateParas:templateParas}=appConfig.cfg,browserConfigs=Array.isArray(appConfig.cfg.browserConfigs)&&appConfig.cfg.browserConfigs.length>0?appConfig.cfg.browserConfigs:[{browserControllerType:browserControllerType,browserType:browserType,proxyUrl:""}];this.#config={useStealthPlugin:useStealthPlugin,browserControllerType:browserControllerType,browserType:browserType,lsdLaunchOptions:lsdLaunchOptions,lsdConnectOptions:lsdConnectOptions,browserConfigs:browserConfigs,maxConcurrency:maxConcurrency,templateParas:templateParas},this.#controller=import_controller4.controller,["playwright","puppeteer"].includes(browserControllerType),this.#browsers=[],this.#browserContextInfos=[]}async init(){const{browserControllerType:defaultBrowserControllerType,browserType:defaultBrowserType,lsdLaunchOptions:lsdLaunchOptions,lsdConnectOptions:lsdConnectOptions}=this.#config;for(const browserConfig of this.#config.browserConfigs){const{browserControllerType:browserControllerType=defaultBrowserControllerType,browserType:browserType=defaultBrowserType,browserUrl:browserUrl,proxyUrl:proxyUrl=""}=browserConfig,proxy=this.#convertProxy(proxyUrl);if(browserUrl){const options=Object.assign({},lsdConnectOptions,{browserUrl:browserUrl}),browser=await this.#controller.connect(browserControllerType,browserType,options);this.#browsers.push(browser);for(const bc of browser.browserContexts())await this.#addBrowserContext(bc,proxy)}else if("string"==typeof proxyUrl){const browserProxy=proxyUrl?proxy:void 0,options=Object.assign({},lsdLaunchOptions,{proxy:browserProxy}),browser=await this.#controller.launch(browserControllerType,browserType,options);this.#browsers.push(browser);for(const bc of browser.browserContexts())await this.#addBrowserContext(bc,proxy)}}return await(0,import_utils12.sleep)(2e3),!0}#convertProxy(proxyUrl,proxyId="",username="",password=""){return{proxyUrl:proxyUrl,proxyId:proxyId\|\|proxyUrl,username:username,password:password}}async getTaskNetworkContext(templateId=0){if(0===this.#browserContextInfos.length)throw new Error("please call init() first to launch or connect browsers.");let templateMapValue;const randomIdxes=(0,import_utils12.getRandomArrayIdx)(this.#browserContextInfos.length);for(const idx of randomIdxes){const browserContextInfo=this.#browserContextInfos[idx],{proxy:proxy,browserContext:browserContext,browserApiContext:browserApiContext,standaloneApiContext:standaloneApiContext,templateMap:templateMap}=browserContextInfo;if(templateId>0&&(templateMapValue=templateMap.get(templateId),templateMapValue\|\|(templateMapValue={templateId:templateId,maxConcurrency:this.#getMaxConcurrency(templateId),running:0},templateMap.set(templateId,templateMapValue)),templateMapValue.maxConcurrency<=templateMapValue.running))continue;if(!browserContext.hasFreePage())continue;const page=await browserContext.getPage();return page?(templateId>0&&templateMapValue&&templateMapValue.running++,page.setPageInfo({relatedId:templateId}),{proxy:proxy,page:page,browserApiContext:browserApiContext,standaloneApiContext:standaloneApiContext}):null}return null}async freePage(page){const templateId=page.pageInfo().relatedId;if(templateId>0){const browserContextId=page.browserContext().id(),browserContextInfo=this.#browserContextInfos.find((bci=>bci.browserContext.id()===browserContextId));if(!browserContextInfo)throw new Error("No browserContext has the page to free.");const templateMapInfo=browserContextInfo.templateMap.get(templateId);if(!templateMapInfo)throw new Error(`No templateMapInfo has templateId ${templateId} in ${browserContextId}.`);templateMapInfo.running--}return await page.free(),!0}},fsp3=__toESM(require("fs/promises"),1),path7=__toESM(require("path"),1),import_utils13=require("@letsscrapedata/utils"),SchedulerInScraper=class _SchedulerInScraper{#taskQueueMap;#taskDIr;#exitWhenCompleted;#browserManager;runningTaskSet;static#newLineChar="\n";static#tryingFlag=!1;constructor(browserManager){this.#taskQueueMap=new Map,this.#taskDIr=appConfig.cfg.taskDir,this.#exitWhenCompleted=appConfig.cfg.exitWhenCompleted,this.#browserManager=browserManager,this.runningTaskSet=new Set}async#readParasStrsFromeFile(filename,deleteFile=!1){try{const lines=(await fsp3.readFile(filename,"utf8")).split(_SchedulerInScraper.#newLineChar),parasStrSet=new Set(lines);return parasStrSet.size>1&&parasStrSet.has("")&&parasStrSet.delete(""),deleteFile&&await fsp3.unlink(filename),Array.from(parasStrSet.values())}catch(err){return[]}}static#templateLockMap=new Map;async#writeTasksIntoFile(parasStrs,templateId,failed=!1){if(_SchedulerInScraper.#templateLockMap.get(templateId)){for(let i=0;i<50&&(await(0,import_utils13.sleep)(200),_SchedulerInScraper.#templateLockMap.get(templateId));i++);if(_SchedulerInScraper.#templateLockMap.get(templateId))return!1}_SchedulerInScraper.#templateLockMap.set(templateId,!0);try{if(failed){const filename=path7.join(this.#taskDIr,`failedtempl${templateId}.txt`);return await fsp3.appendFile(filename,parasStrs.join(_SchedulerInScraper.#newLineChar).concat(_SchedulerInScraper.#newLineChar)),!0}{const filename=path7.join(this.#taskDIr,`tid${templateId}.txt`);return parasStrs.length>0?await fsp3.writeFile(filename,parasStrs.join(_SchedulerInScraper.#newLineChar)):await(0,import_utils13.isFile)(filename)&&await fsp3.unlink(filename),_SchedulerInScraper.#templateLockMap.delete(templateId),!0}}catch(err){return logwarn(`Failed to write task queue of tempatedId ${templateId} into file`),_SchedulerInScraper.#templateLockMap.delete(templateId),!1}}async#loadFailedTasks(){const taskDir=appConfig.cfg.taskDir;if(!taskDir\|\|!(0,import_utils13.isDirectory)(taskDir))return!1;const filenames=await(0,import_utils13.filesInDir)(taskDir,"general","failedtempl","txt");for(const filename of filenames){const templateId=parseInt(filename.slice(11,-4)),parasStrs=await this.#readParasStrsFromeFile(path7.join(taskDir,filename),!0);if(parasStrs.length>0){loginfo(`load unfinished ${parasStrs.length} tasks of templateId ${templateId}`);const queue=this.#taskQueueMap.get(templateId);queue?parasStrs.forEach((parasStr=>queue.push(parasStr))):this.#taskQueueMap.set(templateId,parasStrs)}}return!0}async#loadUnfinishedTasks(loadFailedTasks=!1){const taskDir=appConfig.cfg.taskDir;if(!taskDir\|\|!(0,import_utils13.isDirectory)(taskDir))return!1;const filenames=await(0,import_utils13.filesInDir)(taskDir,"general","templ","txt");for(const filename of filenames){const templateId=parseInt(filename.slice(5,-4)),parasStrs=await this.#readParasStrsFromeFile(path7.join(taskDir,filename));if(parasStrs.length>0){loginfo(`load unfinished ${parasStrs.length} tasks of templateId ${templateId}`);const queue=this.#taskQueueMap.get(templateId);queue?parasStrs.forEach((parasStr=>queue.push(parasStr))):this.#taskQueueMap.set(templateId,parasStrs)}}return loadFailedTasks&&await this.#loadFailedTasks(),!0}async addNewTask(newTask,tryToExecute){const{tid:tid,parasstrs:parasstrs}=newTask;if(tid<=0\|\|0===parasstrs.length)return!1;let queue=this.#taskQueueMap.get(tid);return queue\|\|(queue=[],this.#taskQueueMap.set(tid,queue)),parasstrs.forEach((p=>{queue.includes(p)\|\|queue.push(p)})),await this.#writeTasksIntoFile(queue,tid),tryToExecute&&await this.tryToExecuteTasks([tid]),!0}async init(newTasks=[]){appConfig.cfg.moveDataWhenStart&&await moveDataFiles();const loadFailedTasks=appConfig.cfg.loadFailedTasksInterval>0;appConfig.cfg.loadUnfinishedTasks&&await this.#loadUnfinishedTasks(loadFailedTasks);for(const task of newTasks){const{tid:tid,parasstrs:parasstrs}=task;this.addNewTask({tid:tid,parasstrs:parasstrs},!1)}return await this.tryToExecuteTasks(),setInterval(this.tryToExecuteTasks.bind(this),6e4),loadFailedTasks&&setInterval(this.#loadFailedTasks.bind(this),6e4appConfig.cfg.loadFailedTasksInterval),!0}#getTaskIdStr(templateId,parasStr){return`${templateId}@@${parasStr}`}async#tryToExecuteOneTask(templateId,parasStr,taskNetworkContext){const taskResult=await performOneTask(templateId,parasStr,taskNetworkContext,"indAsync","",0),{credits:credits,subtasks:subtasks}=taskResult.taskData;let templateIdSet=new Set;const parasStrs=this.#taskQueueMap.get(templateId);if(parasStrs){const idx=parasStrs.findIndex((ps=>ps===parasStr));if(idx<0?logerr(`Task queue of templateId ${templateId} has no parasStr: ${parasStr}`):parasStrs.splice(idx,1),credits>=-1){if(0===parasStrs.length&&this.#taskQueueMap.delete(templateId),subtasks.length>0){templateIdSet=new Set(subtasks.map((s=>s.tid)));for(const tid of templateIdSet){const parasstrs=subtasks.filter((s=>s.tid===tid)).map((s=>s.parasstr));await this.addNewTask({tid:tid,parasstrs:parasstrs},!1)}}await this.#writeTasksIntoFile(parasStrs,templateId)}else await this.#writeTasksIntoFile([parasStr],templateId,!0);this.runningTaskSet.delete(this.#getTaskIdStr(templateId,parasStr))}else logerr(`No task queue of templateId ${templateId}`);return taskNetworkContext.page&&await this.#browserManager.freePage(taskNetworkContext.page),appConfig.cfg.minMiliseconds>0&&await(0,import_utils13.sleep)(appConfig.cfg.minMiliseconds),await this.tryToExecuteTasks([templateId],1),templateIdSet.size>0&&await this.tryToExecuteTasks(),!0}async tryToExecuteTasks(templateIds=[],maxNum=0){if(loginfo(`---- tryToExecuteTasks templateIds ${templateIds.join(",")} : `,this.#taskQueueMap),_SchedulerInScraper.#tryingFlag)return logdbg("---- another tryToExecuteTasks is running now"),!0;this.#exitWhenCompleted&&0===this.#taskQueueMap.size&&process.exit(0),_SchedulerInScraper.#tryingFlag=!0;try{let launchedNum=0;0===templateIds.length&&(templateIds=Array.from(this.#taskQueueMap.keys())),logdbg(`templateIds: ${templateIds.join(",")}`);for(const templateId of templateIds){const parasStrs=this.#taskQueueMap.get(templateId);if(!parasStrs)break;const len=parasStrs.length;for(let i=0;i<len;i++){const parasStr=parasStrs[i],taskIdStr=this.#getTaskIdStr(templateId,parasStr);if(this.runningTaskSet.has(taskIdStr))continue;const taskNetworkContext=await this.#browserManager.getTaskNetworkContext(templateId);if(!taskNetworkContext){logdbg(`No page to execute ${taskIdStr}`);break}if(this.runningTaskSet.add(taskIdStr),this.#tryToExecuteOneTask(templateId,parasStr,taskNetworkContext),launchedNum++,maxNum>0&&launchedNum>=maxNum)return _SchedulerInScraper.#tryingFlag=!1,!0}}return _SchedulerInScraper.#tryingFlag=!1,!0}catch(err){return logdbg(err),_SchedulerInScraper.#tryingFlag=!1,!1}}},import_utils15=require("@letsscrapedata/utils"),import_events=__toESM(require("events"),1),import_utils14=require("@letsscrapedata/utils"),TwoCaptcha=class{#clientKey;#status;#pollingInterval;#timeout;#callbackUrl;#supportedCaptchaTypes;#eventBus;async#fetch(url,body,method="POST"){try{const options={method:method,headers:{"Content-Type":"application/json"},body:JSON.stringify(body)},response=await fetch(url,options);if(!response.ok)return logerr(`##captcha failed to fetch, url ${url}`),null;return await response.json()}catch(err){return logerr(err),null}}async#createTask(task){const body={clientKey:this.#clientKey,task:task,softId:0};"string"==typeof task.lang&&task.lang&&(body.languagePool=task.lang,delete task.lang);const res=await this.#fetch("https://api.2captcha.com/createTask",body);return 10===res.errorId?this.setStatus("nobalance"):[11,21,55].includes(res.errorId)&&this.setStatus("disabled"),res}async#getResult(taskId){const body={clientKey:this.#clientKey,taskId:taskId};return await this.#fetch("https://api.2captcha.com/getTaskResult",body)}async#getBalance(){const body={clientKey:this.#clientKey},res=await this.#fetch("https://api.2captcha.com/getBalance",body);return res?.balance>=0?res.balance:-1}#getEventName(taskId){return`event${taskId}`}#getMinWaitMs(_captchaType){return 5e3}async#executeTask(task,captchaType){const response={errorCode:"SUCCEEDED",taskId:0,cost:0,ip:"",createTime:(0,import_utils14.getCurrentUnixTime)(),endTime:0,solveCount:0},{errorId:errorId,taskId:taskId}=await this.#createTask(task);if(errorId>0\|\|taskId<=0)return response.errorCode="ERROR_OTHER",taskId>0&&(response.taskId=taskId),response;response.taskId=taskId;const eventName=this.#getEventName(taskId);await(0,import_utils14.sleep)(this.#getMinWaitMs(captchaType));let interval=0;this.#callbackUrl\|\|(interval=setInterval((async()=>{const res=await this.#getResult(taskId);0!==res?.errorId?this.#eventBus.emit(eventName,{code:1001,message:"failed",data:res}):0===res.errorId&&"ready"===res.status&&this.#eventBus.emit(eventName,{code:0,message:"succeeded",data:res})}),this.#pollingInterval)),setTimeout((()=>this.#eventBus.emit(eventName,{code:1009,message:"timeout",data:null})),this.#timeout);const[eventData]=await(0,import_events.once)(this.#eventBus,eventName);interval&&clearInterval(interval),response.endTime=(0,import_utils14.getCurrentUnixTime)();const{code:code,data:data}=eventData;if(0===code){if(!data\|\|0!==data.errorId)throw new Error("##captcha invalid data in getResult");const{cost:cost,ip:ip,createTime:createTime,endTime:endTime,solveCount:solveCount,solution:solution}=data;cost&&(response.cost=Number(cost)),ip&&(response.ip=ip),createTime&&createTime>0&&(createTime-response.createTime>5&&loginfo("##captcha task createTime difference: "+(createTime-response.createTime)),response.createTime=createTime),endTime&&endTime>0&&(response.endTime-endTime>2this.#pollingInterval&&loginfo("##captcha task endTime difference: "+(response.endTime-endTime)),response.endTime=endTime),solveCount&&solveCount>0&&(response.solveCount=solveCount),solution&&(response.solution=solution)}else if(1001===code)response.errorCode=data?.errorCode?data.errorCode:"ERROR_OTHER";else{if(1009!==code)throw new Error(`##captcha unreachable at ${(new Error).stack}`);response.errorCode="ERROR_TIMEOUT"}return response}#convertTask(origTask,taskName,proxy){let type=proxy?taskName:`${taskName}Proxyless`;return proxy?Object.assign({},origTask,{type:type},proxy):Object.assign({},origTask,{type:type})}#convertGeetestTask(origTask,proxy){const{version:version,websiteURL:websiteURL,gt:gt,challenge:challenge,geetestApiServerSubdomain:geetestApiServerSubdomain,userAgent:userAgent,captchaId:captchaId}=origTask,task={type:proxy?"GeeTestTask":"GeeTestTaskProxyless",version:version,websiteURL:websiteURL};return gt&&(task.gt=gt),challenge&&(task.challenge=challenge),geetestApiServerSubdomain&&(task.geetestApiServerSubdomainapiDomain=geetestApiServerSubdomain),captchaId&&(task.initParameters={captcha_id:captchaId}),userAgent&&(task.userAgent=userAgent),task}#convertRecaptchaTask(origTask,proxy){const{version:version,websiteKey:websiteKey,websiteURL:websiteURL,isEnterprise:isEnterprise,isInvisible:isInvisible,s:s,action:action,minScore:minScore,userAgent:userAgent,cookies:cookies,apiDomain:apiDomain}=origTask;let type="RecaptchaV3TaskProxyless";if(2===version){const taskName=isEnterprise?"RecaptchaV2EnterpriseTask":"RecaptchaV2Task";type=proxy?taskName:`${taskName}Proxyless`}const task={type:type,websiteURL:websiteURL,websiteKey:websiteKey};return apiDomain&&(task.apiDomain=apiDomain),task.isInvisible=isInvisible,s&&(isEnterprise?task.enterprisePayload=s:task.recaptchaDataSValue=s),userAgent&&(task.userAgent=userAgent),cookies&&(task.cookies=cookies),task.isEnterprise=isEnterprise,minScore&&(task.minScore=minScore),action&&(task.pageAction=action),task}#convertTurnstileTask(origTask,proxy){let type=proxy?"TurnstileTask":"TurnstileTaskProxyless";const{websiteKey:websiteKey,websiteURL:websiteURL,action:action,cData:cData,chlPageData:chlPageData}=origTask;return action?{type:type,websiteKey:websiteKey,websiteURL:websiteURL,action:action,data:cData,pagedata:chlPageData}:{type:type,websiteKey:websiteKey,websiteURL:websiteURL}}#convertTextTask(origTask){const{images:images,comment:comment,lang:lang,math:math=!1,maxLength:maxLength=0,minLength:minLength=0,question:question=!1,space:space=!1,textType:textType="any"}=origTask;if(question)return{type:"TextCaptchaTask",comment:comment,lang:lang};{let numeric=0;"numbers"===textType?numeric=1:"letters"===textType&&(numeric=2);const task={type:"ImageToTextTask",body:images[0].toString("base64"),case:!!origTask.case,comment:comment,math:math,maxLength:maxLength,minLength:minLength,numeric:numeric,phrase:space,lang:lang};return images.length>1&&(task.imgInstructions=images[1].toString("base64")),task}}#convertCoordinateTask(origTask){const{images:images,comment:comment,lang:lang,maxClicks:maxClicks=0,minClicks:minClicks=1}=origTask,task={type:"CoordinatesTask",body:images[0].toString("base64"),comment:comment,lang:lang,minClicks:minClicks};return maxClicks>0&&maxClicks>=minClicks&&(task.maxClicks=maxClicks),images.length>1&&(task.imgInstructions=images[1].toString("base64")),task}constructor(options){const{clientKey:clientKey,callbackUrl:callbackUrl="",pollingInterval:pollingInterval=5e3,timeout:timeout=15e4}=options;if(!clientKey)throw new Error("Invalid options: no clientKey");this.#clientKey=clientKey,this.#status="normal",this.#pollingInterval=pollingInterval>=2e3?pollingInterval:2e3,this.#timeout=timeout>=3e4?timeout:3e4,this.#callbackUrl=callbackUrl\|\|"",callbackUrl&&loginfo(this.#callbackUrl),this.#supportedCaptchaTypes=["amazon","funcaptcha","geetest","keycaptcha","mtcaptcha","recaptcha","turnstile","text","coordinate","grid","slider","rotation"],this.#eventBus=new import_events.default}canSolveCaptcha(captchaType){return this.#supportedCaptchaTypes.includes(captchaType)}async getBalance(){return await this.#getBalance()}name(){return"2captcha"}sendResult(result){if(!result\|\|"object"!=typeof result)return!1;const{taskId:taskId,errorId:errorId}=result;if(taskId>0){const eventName=this.#getEventName(result.taskId);return 0===errorId?this.#eventBus.emit(eventName,{code:0,message:"succeeded",data:result}):0===errorId&&this.#eventBus.emit(eventName,{code:1001,message:"failed",data:result}),!0}return!1}status(){return this.#status}setStatus(newStatus){switch(newStatus){case"normal":case"disabled":case"nobalance":this.#status=newStatus;break;default:(0,import_utils14.unreachable)(newStatus)}return!0}async amazon(task,proxy){return await this.#executeTask(this.#convertTask(task,"AmazonTask",proxy),"amazon")}async funcaptcha(task,proxy){return await this.#executeTask(this.#convertTask(task,"FunCaptchaTask",proxy),"funcaptcha")}async geetest(task,proxy){return await this.#executeTask(this.#convertGeetestTask(task,proxy),"geetest")}async keycaptcha(task,proxy){return await this.#executeTask(this.#convertTask(task,"KeyCaptchaTask",proxy),"keycaptcha")}async mtcaptcha(task,proxy){return await this.#executeTask(this.#convertTask(task,"MtCaptchaTask",proxy),"mtcaptcha")}async recaptcha(task,proxy){return await this.#executeTask(this.#convertRecaptchaTask(task,proxy),"recaptcha")}async turnstile(task,proxy){return await this.#executeTask(this.#convertTurnstileTask(task,proxy),"turnstile")}async text(task){return await this.#executeTask(this.#convertTextTask(task),"text")}async coordinate(task){return await this.#executeTask(this.#convertCoordinateTask(task),"coordinate")}async grid(task){return await this.#executeTask(this.#convertTask(task,"TextTask"),"text")}async slider(task){return await this.#executeTask(this.#convertTask(task,"TextTask"),"text")}async rotation(task){return await this.#executeTask(this.#convertTask(task,"TextTask"),"text")}};async function updateScraperConfig(config){const appCfg=appConfig.cfg,{baseDir:baseDir,inputFileDirPart:inputFileDirPart,useStealthPlugin:useStealthPlugin,browserControllerType:browserControllerType,browserType:browserType,lsdLaunchOptions:lsdLaunchOptions,lsdConnectOptions:lsdConnectOptions,browserConfigs:browserConfigs,urlPrefix:urlPrefix,maxConcurrency:maxConcurrency,readCode:readCode,templateParas:templateParas,totalMaxConcurrency:totalMaxConcurrency,minMiliseconds:minMiliseconds,exitWhenCompleted:exitWhenCompleted,useParasstrInXmlIfNeeded:useParasstrInXmlIfNeeded,loadUnfinishedTasks:loadUnfinishedTasks,loadFailedTasksInterval:loadFailedTasksInterval,moveDataWhenStart:moveDataWhenStart,dataFileFormat:dataFileFormat,columnSeperator:columnSeperator}=config;if(baseDir)appCfg.baseDir=baseDir;else{const cwd=process.cwd();appCfg.baseDir=path8.join(cwd,"scraperdata")}if(await(0,import_utils15.fsCheckOrCreateDir)(appCfg.baseDir,!0,!0),appCfg.baseDownloadDir=path8.join(appCfg.baseDir,"downloads"),await(0,import_utils15.fsCheckOrCreateDir)(appCfg.baseDownloadDir,!0,!0),appCfg.baseFileDir=path8.join(appCfg.baseDir,"files"),await(0,import_utils15.fsCheckOrCreateDir)(appCfg.baseFileDir,!0,!0),config.templateDir?appCfg.templateDir=config.templateDir:appCfg.templateDir=path8.join(appCfg.baseDir,"template"),await(0,import_utils15.fsCheckOrCreateDir)(appCfg.templateDir,!0,!0),appCfg.taskDir=path8.join(appCfg.baseDir,"task"),await(0,import_utils15.fsCheckOrCreateDir)(appCfg.taskDir,!0,!0),appCfg.exportsDir=path8.join(appCfg.baseDir,"exports"),await(0,import_utils15.fsCheckOrCreateDir)(appCfg.exportsDir,!0,!0),inputFileDirPart&&(appCfg.inputFileDirPart=inputFileDirPart),browserControllerType&&(appCfg.browserControllerType=browserControllerType),browserType&&(appCfg.browserType=browserType),lsdLaunchOptions&&"object"==typeof lsdLaunchOptions){appCfg.lsdLaunchOptions=Object.assign(appCfg.lsdLaunchOptions,lsdLaunchOptions);const addons=appCfg.lsdLaunchOptions.addons;Array.isArray(addons)&&addons.length>0&&(appCfg.lsdLaunchOptions.addons=addons.map((addon=>path8.isAbsolute(addon)?addon:path8.join(process.cwd(),addon))))}if(lsdConnectOptions&&"object"==typeof lsdConnectOptions&&(appCfg.lsdConnectOptions=Object.assign(appCfg.lsdConnectOptions,lsdConnectOptions)),browserConfigs&&(appCfg.browserConfigs=browserConfigs),urlPrefix&&(appCfg.urlPrefix=urlPrefix),maxConcurrency&&(appCfg.maxConcurrency=maxConcurrency),readCode&&(appCfg.readCode=readCode),templateParas&&(appCfg.templateParas=templateParas),totalMaxConcurrency&&(appCfg.totalMaxConcurrency=totalMaxConcurrency),"number"==typeof minMiliseconds&&(appCfg.minMiliseconds=minMiliseconds),"boolean"==typeof useStealthPlugin&&(appCfg.useStealthPlugin=useStealthPlugin),"boolean"==typeof exitWhenCompleted&&(appCfg.exitWhenCompleted=exitWhenCompleted),"boolean"==typeof useParasstrInXmlIfNeeded&&(appCfg.useParasstrInXmlIfNeeded=useParasstrInXmlIfNeeded),"boolean"==typeof loadUnfinishedTasks&&(appCfg.loadUnfinishedTasks=loadUnfinishedTasks),"number"==typeof loadFailedTasksInterval&&loadFailedTasksInterval>=0&&(appCfg.loadFailedTasksInterval=loadFailedTasksInterval),"boolean"==typeof moveDataWhenStart&&(appCfg.moveDataWhenStart=moveDataWhenStart),dataFileFormat&&(appCfg.dataFileFormat=dataFileFormat),"csv"===appCfg.dataFileFormat?appCfg.columnSeperator=",":"tsv"===appCfg.dataFileFormat?appCfg.columnSeperator="\t":"txt"===appCfg.dataFileFormat&&columnSeperator&&(appCfg.columnSeperator=columnSeperator),config.captcha){const{clientKey:clientKey,captchas:captchas}=config.captcha;"string"==typeof clientKey&&clientKey&&(appCfg.captcha.clientKey=clientKey),Array.isArray(captchas)&&captchas.every((name=>name&&"string"==typeof name))&&(appCfg.captcha.captchas=captchas)}return!0}async function scraper(newTasks=[],config={}){if(await updateScraperConfig(config),appConfig.cfg?.captcha?.clientKey){const twoCaptcha=new TwoCaptcha({clientKey:appConfig.cfg.captcha.clientKey});await addCapatchaProvider(twoCaptcha)}const browserManager=new BrowserManagerInScraper;await browserManager.init();const scheduler=new SchedulerInScraper(browserManager);return await scheduler.init(newTasks),!0}
1	+ "use strict";var __create=Object.create,__defProp=Object.defineProperty,__getOwnPropDesc=Object.getOwnPropertyDescriptor,__getOwnPropNames=Object.getOwnPropertyNames,__getProtoOf=Object.getPrototypeOf,__hasOwnProp=Object.prototype.hasOwnProperty,__export=(target,all)=>{for(var name in all)__defProp(target,name,{get:all[name],enumerable:!0})},__copyProps=(to,from,except,desc)=>{if(from&&"object"==typeof from\|\|"function"==typeof from)for(let key of __getOwnPropNames(from))__hasOwnProp.call(to,key)\|\|key===except\|\|__defProp(to,key,{get:()=>from[key],enumerable:!(desc=__getOwnPropDesc(from,key))\|\|desc.enumerable});return to},__toESM=(mod,isNodeMode,target)=>(target=null!=mod?__create(__getProtoOf(mod)):{},__copyProps(!isNodeMode&&mod&&mod.__esModule?target:__defProp(target,"default",{value:mod,enumerable:!0}),mod)),__toCommonJS=mod=>__copyProps(__defProp({},"__esModule",{value:!0}),mod),src_exports={};__export(src_exports,{performOneTask:()=>performOneTask,scraper:()=>scraper,setScraperLogFun:()=>setScraperLogFun,updateScraperConfig:()=>updateScraperConfig}),module.exports=__toCommonJS(src_exports);var import_controller=require("@letsscrapedata/controller"),import_proxy=require("@letsscrapedata/proxy"),import_utils=require("@letsscrapedata/utils"),pkgLog=import_utils.log;function setScraperLogFun(logFun){return"function"==typeof logFun&&(pkgLog=logFun,(0,import_controller.setControllerLogFun)(pkgLog),(0,import_proxy.setProxyLogFun)(pkgLog),!0)}async function logdbg(...args){await pkgLog(import_utils.LogLevel.DBG,...args)}async function loginfo(...args){await pkgLog(import_utils.LogLevel.INF,...args)}async function logwarn(...args){await pkgLog(import_utils.LogLevel.WRN,...args)}async function logerr(...args){await pkgLog(import_utils.LogLevel.ERR,...args)}(0,import_controller.setControllerLogFun)(pkgLog),(0,import_proxy.setProxyLogFun)(pkgLog),(0,import_utils.setLogLevel)(import_utils.LogLevel.DBG);var import_utils11=require("@letsscrapedata/utils"),import_xml2js_lsd=__toESM(require("xml2js-lsd"),1),defaultSplitOfParases="[\r\n\t,]+",defaultSeperatorOfParas="::",newLineChar="\n",defaultElementCfg={template:{tid:void 0,version:void 0,commited:!1},attrs:{},attr:{name:void 0,value:void 0},paras:{},para:{paraname:void 0,name:"",desc:"",uitype:"",defval:"",listid:"0",listparas:"",min:"",max:"",pattern:"",alert:""},element:{loc:void 0,source:"default",absolute:!1},iframe:{srcprefix:"",loc:""},templstr:{templ:void 0},elecontent_attr:{type:void 0,attrname:void 0,attrname2:"",attrname3:"",attrname4:"",trim:!1,pattern:"",boolattr:!1,multieles:!0,join:newLineChar,line:!1,absolute:!1},elecontent_imgbase64:{type:void 0},elecontent_innerhtml:{type:void 0},elecontent_length:{type:void 0},elecontent_outerhtml:{type:void 0},elecontent_text:{type:void 0,trim:!0,multieles:!0,join:newLineChar,line:!1,top:!1},decodefontsvg:{outtype:"text"},decodefontttf:{fflocs:"",intype:"outerhtml",outtype:"text"},ocrcfg_api:{type:void 0,nlchars:"",seperator:!1,lang:"zh"},ocrcfg_tesseract:{type:void 0,nlchars:"",seperator:!1,lang:"zh"},actions:{},action_api:{type:void 0,method:"GET",url:void 0,encodeuri:!1,session:!1,headerssource:"headers",headers:"",referer:"",proxy:!0,timeout:"30000",context:"default",data:"",datatype:"object",form:!1,path:"",diskcache:!1,cache:!0,tabname:"none",varname:"",errname:"normal",valerrname:"ignore",pattern:"",flags:"",id:"0"},action_break:{type:void 0,id:"0"},action_captcha:{type:void 0,try:"1",errname:"captchafailed",debug:!1,id:"0"},action_click:{type:void 0,wait:"-1",gen:!0,cap:!0,clicktype:"click",try:"1",errname:"cfginvalid",popuppage:"ignored",popupsubtask:!1,login:!0,captcha:!0,navigate:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",downloadpath:"",filename:"",pathvarname:"",id:"0"},action_continue:{type:void 0,id:"0"},action_misc:{type:void 0,id:"0"},misc_delsyncdbdata:{type:void 0,tids:"",retention:"2592000",minretention:"86400"},misc_extractdata:{type:void 0,name:void 0,datatableids:"",method:"put",paras:!0,execdata:!0,subtasks:!0,credits:!0},misc_getstatedata:{type:void 0,cookies:"new",localstorage:"new",headers:"none",userdata:"none"},action_exit:{type:void 0,errname:void 0,id:"0"},action_extract:{type:void 0,tabname:"dat_0000000000009911",id:"0"},action_extract_array:{type:void 0,subtype:void 0,list:void 0,requrl:"",subkeys:"",keys:"",tabname:"dat_0000000000009921",varname:"",idxname:"",maxloops:"0",errname:"cfginvalid",id:"0"},action_extract_script:{type:void 0,subtype:void 0,desc:"",base64:!1,html:!1,isolated:!0,tabname:"dat_0000000000009931",maxloops:"0",errname:"cfginvalid",id:"0"},action_extract_table:{type:void 0,subtype:void 0,orientation:"horizontal",check:!0,tabname:"dat_0000000000009941",keyformat:"camel",colnames:"",maxloops:"0",errname:"cfginvalid",id:"0"},column_element:{type:void 0,colname:void 0,completed:!1,setvar:!1,datatype:"string",errname:"ignore",valerrname:"ignore",pattern:"",flags:"",id:"0",discarded:!1},column_property:{type:void 0,subkeys:void 0,colname:void 0,completed:!1,setvar:!1,datatype:"string",valerrname:"ignore",pattern:"",flags:"",id:"0",discarded:!1},column_ocr:{type:void 0,colname:void 0,completed:!1,setvar:!1,datatype:"string",valerrname:"ignore",pattern:"",flags:"",id:"0",discarded:!1},column_subtask:{type:void 0,colname:void 0,completed:!1,setvar:!1,datatype:"string",valerrname:"ignore",pattern:"",flags:"",id:"0",discarded:!1},column_templstr:{type:void 0,colname:void 0,completed:!1,setvar:!1,datatype:"string",valerrname:"ignore",pattern:"",flags:"",id:"0",discarded:!1},action_goto:{type:void 0,url:void 0,reuse:!1,wait:"-1",encodeuri:!1,gen:!0,cap:!0,popuppage:"ignored",popupsubtask:!1,login:!0,captcha:!0,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",headers:"",referer:"",id:"0"},action_hover:{type:void 0,try:"1",gen:!1,cap:!1,wait:"-1",errname:"cfginvalid",id:"0"},action_ifelse:{type:void 0,id:"0"},condition_element:{type:void 0},condition_else:{type:void 0},condition_templstr:{type:void 0},action_input:{type:void 0,content:void 0,enter:!1,replace:!1,gen:!1,cap:!1,try:"1",wait:"-1",errname:"cfginvalid",popuppage:"ignored",popupsubtask:!1,login:!1,captcha:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",id:"0"},action_intercept_clear:{type:void 0,subtype:void 0},action_intercept_set:{type:void 0,subtype:void 0},request_abort:{type:void 0,method:"all",url:"",resourcetype:"image",postdata:""},response_cache:{type:void 0,method:"all",url:void 0,resourcetype:"all",postdata:"",minsize:"0",maxsize:"0",contenttype:"text",cache:!0,tabname:"none",requestheaders:!1,responseheaders:!1},request_data:{type:void 0,subkeys:void 0,value:void 0,datatype:void 0},request_header:{type:void 0,name:void 0,value:void 0},response_save:{type:void 0,method:"all",url:void 0,resourcetype:"all",postdata:"",minsize:"0",maxsize:"0",basedir:"response",pathtype:"path_name",hashmethod:"MD5",hostnameparts:"0",extname:"",validextnames:"",params:"",encoding:"",tabname:"none"},response_statuscode:{type:void 0,codes:void 0,errname:void 0},action_loopdowhile_element:{type:void 0,subtype:void 0,iswhile:!1,varname:"",maxloops:"0",click:!1,navigate:!0,gen:!0,cap:!0,errname:"cfginvalid",wait:"-1",id:"0"},action_loopdowhile_templstr:{type:void 0,subtype:void 0,iswhile:!1,varname:"",maxloops:"0",id:"0"},action_loopfor:{type:void 0,from:void 0,to:void 0,step:"1",roundtype:"ceil",varname:void 0,maxloops:"0",errname:"ignore",id:"0"},action_loopineles:{type:void 0,varname:"",maxloops:"0",start:"1",end:"0",step:"1",errname:"ignore",id:"0"},action_loopinstr:{type:void 0,list:void 0,split:defaultSplitOfParases,varname:void 0,maxloops:"0",trim:!0,errname:"ignore",id:"0"},action_scroll_by:{type:void 0,subtype:void 0,height:"100",unit:"windowpct",maxtimes:"1",interval:"800",gen:!1,cap:!1,popuppage:"ignored",popupsubtask:!1,login:!1,captcha:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",id:"0"},action_scroll_intoview:{type:void 0,subtype:void 0,gen:!1,cap:!1,errname:"cfginvalid",popuppage:"ignored",popupsubtask:!1,login:!1,captcha:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",id:"0"},action_scroll_to:{type:void 0,subtype:void 0,height:"100",unit:"bodypct",gen:!1,cap:!1,popuppage:"ignored",popupsubtask:!1,login:!1,captcha:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",id:"0"},action_select:{type:void 0,selecttype:"value",selectval:"",gen:!1,cap:!1,try:"1",wait:"-1",errname:"cfginvalid",popuppage:"ignored",popupsubtask:!1,login:!1,captcha:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",id:"0"},dbquery:{dbname:void 0,sql:void 0,sqlparas:"",parasplit:defaultSeperatorOfParas,join:newLineChar},action_setvar_dbquery:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",errname:"ignore",setkeys:"",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},action_setvar_element:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",try:"1",errname:"ignore",setkeys:"",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},file:{path:void 0,encoding:"utf8"},action_setvar_file:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",proxy:!0,errname:"ignore",setkeys:"",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},action_setvar_get:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",errname:"ignore",setkeys:"",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},action_setvar_ocr:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",errname:"ignore",setkeys:"",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},action_setvar_subtask:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",errname:"ignore",setkeys:"",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},action_setvar_templstr:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",errname:"ignore",setkeys:"",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},subtask:{tid:void 0,parasstr:void 0,idx:"",minlen:"1",errname:"cfginvalid",popuppage:!1},action_subtask:{type:void 0,subtasks:"",sameasparent:!1,id:"0"},action_wait_element:{type:void 0,subtype:void 0,timeout:"30000",state:"visible",errname:"ignore",wait:"-1",id:"0"},action_wait_navigation:{type:void 0,subtype:void 0,timeout:"30000",waituntil:"load",url:"",errname:"cfginvalid",wait:"-1",id:"0"},action_wait_sleep:{type:void 0,subtype:void 0,minms:"5000",maxms:"10000",errname:"ignore",wait:"-1",id:"0"},fonts:{},fontsvg:{exloc:"head link[rel='stylesheet']",inloc:"",csmaptype:"general",bsfilter:""},fontttf:{exloc:"head link[rel='stylesheet']",inloc:"",minuc:"57344",maxuc:"63744",startidx:"-1",fsfilter:"",fufilter:"",parsetype:"none"},fontselector:{name:void 0,fontfamily:void 0},fontfamily:{name:void 0,fontcodes:void 0,fontchars:void 0},fontcodes:{name:void 0,codes:void 0},fontchars:{name:void 0,chars:void 0},myfuns:{},myfun:{name:void 0,desc:"",base64:!1},captcha_amazon:{type:void 0},captcha_funcaptcha:{type:void 0},captcha_geetest:{type:void 0},captcha_keycaptcha:{type:void 0},captcha_mtcaptcha:{type:void 0},captcha_recaptcha:{type:void 0,minscore:"0.3"},captcha_turnstile:{type:void 0},image_element:{type:void 0},comment_element:{type:void 0,attr:"textContent"},input_element:{type:void 0,enter:!1},submit_element:{type:void 0},check_result:{type:void 0,attr:"",failedstr:""},captcha_text:{type:void 0,case:!1,lang:"en",math:!1,maxlen:"0",minlen:"0",question:!1,space:!1,texttype:"any"},captcha_coordinate:{type:void 0,lang:"en",maxclicks:"0",minclicks:"1"},transform:{defaultval:"",usevar:!1},fun_c_attr:{type:void 0,subtype:void 0,loc:void 0,idx:"0",attrname:void 0},fun_c_data:{type:void 0,subtype:void 0,loc:void 0,idx:"0",attrname:void 0},fun_c_html:{type:void 0,subtype:void 0,loc:void 0,idx:"0"},fun_c_length:{type:void 0,subtype:void 0,loc:void 0},fun_c_text:{type:void 0,subtype:void 0,loc:void 0,idx:"-9999"},fun_ceil:{type:void 0},fun_closingsubstr:{type:void 0,openchar:void 0,startstr:"",position:"0"},fun_compress:{type:void 0,method:"gzip",flush:"",finishflush:"",chunksize:"0",windowbits:"0",level:"",memlevel:"",strategy:"",sourceencoding:"utf8",targetencoding:"base64"},fun_concat:{type:void 0,str1:void 0,str2:"",str3:"",str4:"",str5:""},fun_convertencoding:{type:void 0,sourceencoding:"base64",targetencoding:"utf8",start:"0",end:"0"},fun_decompress:{type:void 0,method:"gunzip",flush:"",finishflush:"",chunksize:"0",windowbits:"0",sourceencoding:"base64",targetencoding:"utf8",start:"0",end:"0"},fun_decode:{type:void 0,str1:"",val1:"",str2:"",val2:"",str3:"",val3:"",str4:"",val4:"",str5:"",val5:"",defaultval:""},fun_floor:{type:void 0},fun_hashcode:{type:void 0,hashmethod:"MD5"},fun_includes:{type:void 0,searchstr:void 0},fun_insert:{type:void 0,str:void 0},fun_itemstoobj:{type:void 0,split:defaultSplitOfParases,kvsplit:":",keytrim:!0,keynows:!1,valuetrim:!0,valuenows:!1},fun_jsonparse:{type:void 0,key1:"",key2:"",key3:"",key4:"",key5:"",key6:"",keys:!1,length:!1,srctype:"json"},fun_length:{type:void 0},fun_matchall:{type:void 0,pattern:void 0,flags:"g",itemvalue:"match[0]",filter:"",start:"0",end:"0",resulttype:"all",join:newLineChar},fun_max:{type:void 0,split:defaultSplitOfParases},fun_min:{type:void 0,split:defaultSplitOfParases},fun_numeq:{type:void 0,num:void 0},fun_numge:{type:void 0,num:void 0},fun_numgt:{type:void 0,num:void 0},fun_numle:{type:void 0,num:void 0},fun_numlt:{type:void 0,num:void 0},fun_notincludes:{type:void 0,searchstr:void 0},fun_padend:{type:void 0,targetlen:void 0,padstr:" "},fun_padstart:{type:void 0,targetlen:void 0,padstr:" "},fun_parsefloat:{type:void 0},fun_parseint:{type:void 0},fun_randomarryidxes:{type:void 0,join:newLineChar},fun_replace:{type:void 0,substr:void 0,newstr:void 0,regexp:!1,flags:""},fun_round:{type:void 0},fun_setcomplement:{type:void 0,array2:void 0,split1:defaultSplitOfParases,split2:defaultSplitOfParases,sorttype:"nosort",valtype:"number",limit:"0",join:newLineChar},fun_setdifference:{type:void 0,array2:void 0,split1:defaultSplitOfParases,split2:defaultSplitOfParases,sorttype:"nosort",valtype:"number",limit:"0",join:newLineChar},fun_setintersection:{type:void 0,array2:void 0,split1:defaultSplitOfParases,split2:defaultSplitOfParases,sorttype:"nosort",valtype:"number",limit:"0",join:newLineChar},fun_setunion:{type:void 0,array2:void 0,split1:defaultSplitOfParases,split2:defaultSplitOfParases,sorttype:"nosort",valtype:"number",limit:"0",join:newLineChar},fun_slice:{type:void 0,beginidx:void 0,endidx:""},fun_shuffle:{type:void 0,split:defaultSplitOfParases},fun_sort:{type:void 0,split:defaultSplitOfParases,valtype:"number",sorttype:"asc",join:newLineChar},fun_specialconvert:{type:void 0,method:void 0,para1:"",para2:"",para3:"",para4:"",para5:""},fun_split:{type:void 0,split:defaultSplitOfParases,limit:"0",filter:"",start:"0",end:"0",resulttype:"all",val:"",insertstr:"",join:newLineChar},fun_streq:{type:void 0,str:void 0},fun_substrafter:{type:void 0,substr:void 0,position:"0",last:!1,include:!1},fun_substrbefore:{type:void 0,substr:void 0,position:"0",last:!1,include:!1},fun_tobool:{type:void 0},fun_tolowercase:{type:void 0},fun_tonum:{type:void 0},fun_touppercase:{type:void 0},fun_trim:{type:void 0},fun_trimend:{type:void 0},fun_trimstart:{type:void 0},fun_uniq:{type:void 0,split:defaultSplitOfParases,join:newLineChar},fun_urldelparams:{type:void 0,param1:void 0,param2:"",param3:""},fun_urldecode:{type:void 0,method:"URI",encoding:"utf8",percent:!0},fun_urlencode:{type:void 0,method:"URI",encoding:"utf8",percent:!0},fun_urlget:{type:void 0,attrorpath:void 0,hashmethod:"MD5",hostnameparts:"0",params:"",param:"",extname:"",validextnames:"",title:""},fun_urlsetparams:{type:void 0,param1:void 0,val1:"",param2:"",val2:"",param3:"",val3:"",replace:!0},fun_myfun:{type:void 0,name:void 0,arg1:"",arg2:"",arg3:"",arg4:"",arg5:""},get_addedtaskparas:{type:void 0,tid:void 0,limit:"10000",join:newLineChar,interval:"3600"},get_content:{type:void 0},get_cookies:{type:void 0,urls:"",domain:"",name:"",path:"",value:!1},get_datetime:{type:void 0,format:"yyyy-MM-ddThh:mm:ss"},get_execedtaskparas:{type:void 0,tid:void 0,limit:"10000",join:newLineChar,interval:"3600"},get_file:{type:void 0,url:void 0,path:"",basedir:"files",pathtype:"path_name",hashmethod:"MD5",hostnameparts:"0",params:"",extname:"",validextnames:"",pathvarname:"",proxy:!0,headers:"",referer:"",setvar:!1},httpbrowser:{name:void 0,minversion:"0",maxversion:"0",httpversion:"0"},httpdevice:{device:void 0},httpheader:{name:void 0,source:"value",value:void 0,delete:!1},httpos:{os:void 0},get_httpheaders:{type:void 0,headermethod:"genLatestWindowsChrome",origheaders:"",browserlist:"",httpversion:"0"},get_mhtml:{type:void 0,path:"",basedir:"mhtml",pathtype:"title",hashmethod:"MD5",hostnameparts:"0",pathvarname:""},get_pdf:{type:void 0,path:"",basedir:"pdf",pathtype:"title",hashmethod:"MD5",hostnameparts:"0",scale:"1",displayheaderfooter:!1,headertemplate:"",footertemplate:"",printbackground:!1,landscape:!1,pageranges:"",format:"Letter",width:"",height:"",top:"",right:"",bottom:"",left:"",screen:!1,onepage:!1,hmargin:"",setvar:!1,pathvarname:""},get_queuedcntwithparas:{type:void 0,tid:void 0,parasstr:void 0,operator:"eq"},get_queuedtaskparas:{type:void 0,tid:void 0,limit:"10000",join:newLineChar,interval:"3600"},get_random:{type:void 0,min:void 0,max:void 0},get_response:{type:void 0,requrl:"",idx:"-1",length:!1},get_screenshot:{type:void 0,basedir:"screenshot",path:"",extname:"png",quality:"-1",fullpage:!1,x:"0",y:"0",width:"0",height:"0",omitbackground:!1,setvar:!1,pathvarname:""},get_special:{type:void 0,method:void 0,parasstr:""},get_title:{type:void 0,aspath:!1},get_window:{type:void 0,key1:void 0,key2:"",key3:"",key4:"",key5:"",key6:"",keys:!1,length:!1}},import_controller2=require("@letsscrapedata/controller"),appConfig={XML_ATTR_KEY:"$$",XML_CHAR_KEY:"_",XML_CHILDREN_KEY:"children",GEN_CREDITS_BITS:16,GEN_CREDITS_AND:65535,CHECK_DNS_DOMAIN:"example.com",CHECK_POPUP_PAGE_TIMEOUT1:35e3,CHECK_POPUP_PAGE_TIMEOUT2:5e3,RESERVED_PARA_NAMES:["authInfo","inParas","sysParas"],EVENT_TYPES:{lsdPageClose:import_controller2.LsdPageEvent.PAGE_CLOSE,lsdPagePopup:import_controller2.LsdPageEvent.PAGE_POPUP,lsdScraperPopup:"lsdScraperPopup",taskComplete:"taskComplete"},OPERATE_ACTIONS:["click","goto","hover","input","scroll","select","wait","captcha"],performOneTask:console.log,cfg:{exitWhenCompleted:!1,useParasstrInXmlIfNeeded:!1,loadUnfinishedTasks:!1,loadFailedTasksInterval:0,baseDir:"",baseDownloadDir:"",baseFileDir:"",templateDir:"",taskDir:"",exportsDir:"",inputFileDirPart:"LetsScrapeData",useStealthPlugin:!1,browserControllerType:"patchright",browserType:"chromium",lsdLaunchOptions:{headless:!1,geoip:!0},lsdConnectOptions:{browserUrl:""},browserConfigs:[],captcha:{clientKey:"",captchas:[]},maxConcurrency:1,readCode:"",templateParas:[],urlPrefix:"https://api.letsscrapedata.com/nologin",totalMaxConcurrency:10,minMiliseconds:2e3,moveDataWhenStart:!1,dataFileFormat:"jsonl",columnSeperator:"::"}},TaskParser=class _TaskParser{static#applyDefaultElementCfg(cfg,parentEleName){try{if(!cfg\|\|!parentEleName)throw new Error(`##te Invalid paras, parentEleName ${parentEleName} in _applyDefaultElementCfg`);if("object"!=typeof cfg)return!0;const aKey=appConfig.XML_ATTR_KEY,cKey=appConfig.XML_CHILDREN_KEY;cfg[aKey]\|\|(cfg[aKey]={});let defaultCfgName=parentEleName;cfg[aKey].type&&(defaultCfgName=defaultCfgName+"_"+cfg[aKey].type),cfg[aKey].subtype&&(defaultCfgName=defaultCfgName+"_"+cfg[aKey].subtype);const defaultEleCfg=defaultElementCfg[defaultCfgName];if(!defaultEleCfg)throw new Error(`##te No default Cfg for eleName ${defaultCfgName} in _applyDefaultElementCfg`);const eleJsonCfg=Object.assign({},defaultEleCfg);if(Object.keys(defaultEleCfg).forEach(attrName=>{let attrVal=cfg[aKey][attrName];if("boolean"==typeof eleJsonCfg[attrName])"true"===attrVal\|\|""===attrVal?eleJsonCfg[attrName]=!0:"false"===attrVal&&(eleJsonCfg[attrName]=!1);else if(attrVal\|\|""===attrVal)eleJsonCfg[attrName]=attrVal;else if(void 0===eleJsonCfg[attrName])throw new Error(`##te Attribute ${attrName} of Element ${defaultCfgName} is undefined`)}),cfg[aKey]=eleJsonCfg,!cfg[cKey])return!0;const childKeys=Array.from(Object.keys(cfg[cKey]));for(const childEleName of childKeys){const childs=cfg[cKey][childEleName];1===childs.length&&""===childs[0]?cfg[cKey][childEleName]=[]:childs.forEach(cfg2=>_TaskParser.#applyDefaultElementCfg(cfg2,childEleName))}return!0}catch(err){throw logerr(`##te Error parentEleName ${parentEleName} in _applyDefaultElementCfg`),new Error(err)}}static async convertXmlToJson(xmlStr,defaultCfgFlag=!0){try{const getTagName=function(name){const idx=name.indexOf("_");return idx>0?name.slice(0,idx):name},options={attrkey:appConfig.XML_ATTR_KEY,charkey:appConfig.XML_CHAR_KEY,childkey:appConfig.XML_CHILDREN_KEY,explicitChildren:!0,tagNameProcessors:[getTagName]},cKey=appConfig.XML_CHILDREN_KEY,root=await import_xml2js_lsd.default.parseStringPromise(xmlStr,options);if(!(root&&root.template&&root.template[cKey]&&Array.isArray(root.template[cKey].actions)))throw new Error(`##te Invalid XML config string: ${xmlStr}`);return defaultCfgFlag&&_TaskParser.#applyDefaultElementCfg(root.template,"template"),root.template}catch(err){throw logerr("##te Exeception in convertXmlToJson"),logerr(`##te xmlCfgStr: ${xmlStr}`),new Error(err)}}static getPartOfJsonCfg(jsonCfg,partName,optional=!1){try{const ret=jsonCfg[appConfig.XML_CHILDREN_KEY][partName][0];return ret\|\|null}catch(err){if(optional)return null;throw logerr(`##te Exeception when try to get ${partName} in getPartOfJsonCfg`),new Error(err)}}static#getAttrCfgsFromJsonCfg(jsonCfg){try{const attrs=_TaskParser.getPartOfJsonCfg(jsonCfg,"attrs");return!Array.isArray(attrs?.children?.attr)\|\|attrs.children.attr.length<=0?[]:attrs.children.attr.map(a=>a[appConfig.XML_ATTR_KEY])}catch(err){return logerr(err),[]}}static getParaCfgsFromJsonCfg(jsonCfg){try{const paras=_TaskParser.getPartOfJsonCfg(jsonCfg,"paras");return!Array.isArray(paras?.children?.para)\|\|paras.children.para.length<=0?[]:paras.children.para.map(p=>p[appConfig.XML_ATTR_KEY])}catch(err){return logerr(err),[]}}static getMyfunCfgsFromJsonCfg(jsonCfg){try{const myfuns=_TaskParser.getPartOfJsonCfg(jsonCfg,"myfuns",!0);return!Array.isArray(myfuns?.children?.myfun)\|\|myfuns.children.myfun.length<=0?[]:myfuns.children.myfun.map(p=>Object.assign({},p[appConfig.XML_ATTR_KEY],{script:p[appConfig.XML_CHAR_KEY]}))}catch(err){return logerr(err),[]}}static getAttrsInXml(jsonCfg){const attrConfigs=_TaskParser.#getAttrCfgsFromJsonCfg(jsonCfg),attrsInXml={};return attrConfigs.forEach(attrConfig=>attrsInXml[attrConfig.name]=attrConfig.value),attrsInXml}static#getAllActionsOfType(actions,actionType){const extracts=[],cKey=appConfig.XML_CHILDREN_KEY;for(const action of actions)if(action?.[appConfig.XML_ATTR_KEY]?.type===actionType)extracts.push(action);else if(action&&action[cKey]&&(action[cKey].action\|\|action[cKey].condition)){_TaskParser.#getAllActionsOfType(action[cKey].action?action[cKey].action:action[cKey].condition,actionType).forEach(act=>{extracts.push(act)})}return extracts}static getCaptchTypes(jsonCfg){const cKey=appConfig.XML_CHILDREN_KEY,rootActions=jsonCfg[cKey].actions[0][cKey].action,captchaCfgs=_TaskParser.#getAllActionsOfType(rootActions,"captcha"),captchaTypes=[];for(const captchaCfg of captchaCfgs){const captchas=captchaCfg[appConfig.XML_CHILDREN_KEY]?.captcha;Array.isArray(captchas)&&captchas.forEach(cfg=>{const captchaType=cfg[appConfig.XML_ATTR_KEY].type;captchaType&&captchaTypes.push(captchaType)})}return captchaTypes}static getInParas(parasStr,paraCfgs=[],splitStr="::"){const inParas={},parts=parasStr.split("@@"),prefixes=["para","apara","rpara"];for(let idx=0;idx<3;idx++){let prefix=prefixes[idx],paraNum=paraCfgs.filter(p=>p.paraname.startsWith(prefix)).length;if(parts.length>idx&&parts[idx].length>0){let paras=parts.length>idx&&parts[idx].length>0?parts[idx].split(splitStr):[];for(let i=1;i<=paras.length;i++)inParas[`${prefix}${i}`]=paras[i-1];for(let i=paras.length+1;i<=paraNum;i++)inParas[`${prefix}${i}`]=""}}return inParas}},fs=__toESM(require("fs"),1),path=__toESM(require("path"),1),import_utils2=require("@letsscrapedata/utils"),TemplateManagerInScraper=class _TemplateManagerInScraper{static#templateMap=new Map;static#failedTempateIdSet=new Set;static#getReadCode(templateId){const cfg=appConfig.cfg.templateParas.find(tmc=>tmc.templateId===templateId);return cfg?.readCode?cfg.readCode:appConfig.cfg.readCode}static async#getTemplateFromLocal(templateId,xmlStr){try{if(!xmlStr){if(!appConfig.cfg.templateDir)return null;const filenames=await(0,import_utils2.filesInDir)(appConfig.cfg.templateDir,"general",`tid${templateId}_`,"xml");let filename=1===filenames.length?filenames[0]:`tid${templateId}.xml`;filename=path.join(appConfig.cfg.templateDir,filename),xmlStr=fs.readFileSync(filename,"utf8")}if(!xmlStr)return null;return{templateId:templateId,domainId:1,defaultElementSource:"browser",maxExecutionDuration:600,configDetail:xmlStr}}catch(err){return null}}static async#getTemplateFromServer(templateId,code){try{if(_TemplateManagerInScraper.#failedTempateIdSet.has(templateId))return null;let url=`${appConfig.cfg.urlPrefix}/template`;url=`${url}?tid=${templateId}&code=${code}`;const response=await fetch(url);if(200!==response.status)return 403===response.status?logwarn(`Failed to get template ${templateId}: No privilege`):logwarn(`Failed to get template ${templateId}, status: ${response.status}`),_TemplateManagerInScraper.#failedTempateIdSet.add(templateId),null;const json=await response.json();if(!json\|\|0!==json.code\|\|!json.data)return logwarn(`Failed to get template ${templateId}: ${json.message}`),null;if(json.data.templateId!==templateId)return logerr(`Different templateId ${templateId} / ${json.templateId}`),null;const{domainId:domainId=0,defaultElementSource:defaultElementSource,configDetail:configDetail="",maxExecutionDuration:maxExecutionDuration}=json.data;if(!configDetail)return logerr(`Invalid content in template ${templateId}`),null;return{templateId:templateId,domainId:domainId,defaultElementSource:defaultElementSource,maxExecutionDuration:maxExecutionDuration,configDetail:configDetail}}catch(err){return logdbg(err),null}}static async parseXmlTemplate(xmlStr){const jsonCfg=await TaskParser.convertXmlToJson(xmlStr),paraCfgs=TaskParser.getParaCfgsFromJsonCfg(jsonCfg),fontsConfig=TaskParser.getPartOfJsonCfg(jsonCfg,"fonts",!0),myfunCfgs=TaskParser.getMyfunCfgsFromJsonCfg(jsonCfg),actionConfigs=TaskParser.getPartOfJsonCfg(jsonCfg,"actions",!1),attrsInXml=TaskParser.getAttrsInXml(jsonCfg),captchaTypes=TaskParser.getCaptchTypes(jsonCfg),currentTime=(0,import_utils2.getCurrentUnixTime)(),myfunScripts={};myfunCfgs.forEach(myfunCfg=>{const{name:name,script:script,base64:base64}=myfunCfg;if(base64){const str=(0,import_utils2.decodeFromBase64)(script);"string"==typeof str&&str&&(myfunScripts[name]=str)}else myfunScripts[name]=script});return{actionConfigs:actionConfigs,paraCfgs:paraCfgs,fontsConfig:fontsConfig\|\|null,myfunScripts:myfunScripts,attrsInXml:attrsInXml,captchaTypes:captchaTypes,lastUsedTime:currentTime,lastCheckTime:currentTime}}static async getTemplateConfig(templateId,xmlStr=""){if(!xmlStr){const cachedParsedTemplateExt=_TemplateManagerInScraper.#templateMap.get(templateId);if(cachedParsedTemplateExt)return cachedParsedTemplateExt}let template=await _TemplateManagerInScraper.#getTemplateFromLocal(templateId,xmlStr);if(!template){const code=_TemplateManagerInScraper.#getReadCode(templateId);template=await _TemplateManagerInScraper.#getTemplateFromServer(templateId,code)}if(!template){const message=`Cannot get template tid ${templateId}`;throw logwarn(message),new Error(message)}const parsedTemplate=await _TemplateManagerInScraper.parseXmlTemplate(template.configDetail),parsedTemplateExt=Object.assign({},parsedTemplate,{template:template});return _TemplateManagerInScraper.#templateMap.set(templateId,parsedTemplateExt),parsedTemplateExt}static clearTemplateConfig(templateId=0){return templateId>0?_TemplateManagerInScraper.#templateMap.delete(templateId):_TemplateManagerInScraper.#templateMap.clear(),!0}},import_node_events=__toESM(require("events"),1),import_node_events2=require("events"),fs6=__toESM(require("fs"),1),path5=__toESM(require("path"),1),import_got_scraping=require("got-scraping"),import_isolated_vm2=__toESM(require("isolated-vm"),1),import_utils9=require("@letsscrapedata/utils"),import_controller3=require("@letsscrapedata/controller"),uuid=__toESM(require("uuid"),1),SysParas=class{#taskContext;#templateId;#taskId;#version;constructor(taskContext,templateId,taskId,version){this.#taskContext=taskContext,this.#templateId=templateId,this.#taskId=taskId,this.#version=version}get version(){return this.#version}get templateId(){return this.#templateId}get taskId(){return this.#taskId}get url(){try{return this.#taskContext?.page?this.#taskContext.page.url():""}catch{return""}}get hasPopupPage(){return this.#taskContext?.popupPage?"1":"0"}get subtaskNum(){const subtasks=this.#taskContext?.subtasks;return Array.isArray(subtasks)?String(subtasks.length):"0"}get datatableNum(){const execData=this.#taskContext?.execData;return execData&&"object"==typeof execData?String(Object.keys(execData).length):"0"}get responses(){try{const responses=this.#taskContext?.responses;return Array.isArray(responses)?JSON.stringify(responses):""}catch(err){return""}}get currentDate(){return(new Date).toISOString().slice(0,10)}get currentDateTime(){return(new Date).toISOString().slice(0,19)}get currentIsoTime(){return(new Date).toISOString()}get currentLocalTime(){const date=new Date;return`${date.getFullYear()}-${String(date.getMonth()+1).padStart(2,"0")}-${String(date.getDate()).padStart(2,"0")} ${String(date.getHours()).padStart(2,"0")}:${String(date.getMinutes()).padStart(2,"0")}:${String(date.getSeconds()).padStart(2,"0")}`}get currentUtcTime(){return(new Date).toISOString().slice(0,19).replace("T"," ")}get currentTime(){return(new Date).toISOString().slice(11,19)}get currentUnixMs(){return String(Date.now())}get currentUnixSec(){return String(Math.floor(Date.now()/1e3))}get random10(){return String(Math.floor(10Math.random()))}get random100(){return String(Math.floor(100Math.random()))}get random1000(){return String(Math.floor(1e3Math.random()))}get random10000(){return String(Math.floor(1e4Math.random()))}get uuidv1(){return uuid.v1()}get uuidv4(){return uuid.v4()}},iconv=__toESM(require("iconv-lite"),1),zlib=__toESM(require("zlib"),1),cheerio=__toESM(require("cheerio"),1),import_isolated_vm=__toESM(require("isolated-vm"),1),import_utils3=require("@letsscrapedata/utils"),LsdString=class _LsdString{static specialConvertFunObj={};static setSpecialConvertFunObj(specialConvertFunObj){return _LsdString.specialConvertFunObj=specialConvertFunObj,!0}static transformStrSync(origStr,funCfgs,defaultVal){let str=origStr;try{if("string"!=typeof origStr\|\|!Array.isArray(funCfgs))throw new Error("Invalid paras in transformStr");for(const funCfg of funCfgs){if("object"!=typeof funCfg)throw new Error("Invalid funCfg in transformStr");const functionName="c"===funCfg.type?`${funCfg.type}_${funCfg.subtype}`:funCfg.type;let fun=_LsdString._funPerformers[functionName];fun&&"function"==typeof fun?str=fun(str,funCfg):logerr(`Invalid funName ${funCfg.type} in transformStr`)}}catch(err){loginfo(err)}return str\|\|defaultVal}static async transformStr(origStr,funCfgs,defaultVal,myfunScripts){let str=origStr;try{if("string"!=typeof origStr\|\|!Array.isArray(funCfgs))throw new Error("Invalid paras in transformStr");for(const funCfg of funCfgs){if("object"!=typeof funCfg)throw new Error("Invalid funCfg in transformStr");const functionName="c"===funCfg.type?`${funCfg.type}_${funCfg.subtype}`:funCfg.type;if("myfun"!==functionName){let fun=_LsdString._funPerformers[functionName];if(!fun\|\|"function"!=typeof fun){logerr(`Invalid funName ${funCfg.type} in transformStr`);continue}str=fun(str,funCfg)}else str=await _LsdString.#myfun(str,funCfg,myfunScripts)}}catch(err){loginfo(err)}return str\|\|defaultVal}static#getStdFunScriptFun=null;static setGetStdFunScript(fun){return _LsdString.#getStdFunScriptFun=fun,!0}static async#getScriptOfMyfun(name,myfunScripts){if("string"==typeof name&&name){if(myfunScripts?.[name])return myfunScripts[name];if(name.startsWith("std")&&_LsdString.#getStdFunScriptFun){const script=await _LsdString.#getStdFunScriptFun(name);if(script)return script}}return""}static async#myfun(str,funCfg,myFunScripts){const{name:name,arg1:arg1,arg2:arg2,arg3:arg3,arg4:arg4,arg5:arg5}=funCfg;try{const script=await _LsdString.#getScriptOfMyfun(name,myFunScripts);if(!script)return"";const inData={origStr:str,arg1:arg1,arg2:arg2,arg3:arg3,arg4:arg4,arg5:arg5},isolate=new import_isolated_vm.default.Isolate,context=isolate.createContextSync(),externalCopy=new import_isolated_vm.default.ExternalCopy(inData);context.global.setSync("inData",externalCopy.copyInto());const content=isolate.compileScriptSync(script).runSync(context);return"string"!=typeof content&&logwarn("Please check the script of myfun is correct."),"string"==typeof content?content:""}catch(err){return logwarn(err.message),""}}static __getElementBySelector($,loc,idx=0){try{const eles="root"===loc?$.root():$(loc);return eles.length<=0\|\|-9999===idx?eles:eles.eq(idx)}catch(err){return logerr("##te cheerio invalid $ or loc"),null}}static __cheerioOptions={xmlMode:!0,decodeEntities:!1};static _c_attr(origStr,funCfg){try{const{loc:loc,idx:idx,attrname:attrname}=funCfg,idxInt=parseInt(idx),$=cheerio.load(origStr,_LsdString.__cheerioOptions);return _LsdString.__getElementBySelector($,loc,idxInt).attr(attrname)}catch(err){return logerr(`##tmp cheerio c_attr err: ${err}`),""}}static _c_data(origStr,funCfg){try{const{loc:loc,idx:idx,attrname:attrname}=funCfg,idxInt=parseInt(idx),$=cheerio.load(origStr,_LsdString.__cheerioOptions);return _LsdString.__getElementBySelector($,loc,idxInt).data(attrname)}catch(err){return logerr(`##tmp cheerio c_data err: ${err}`),""}}static _c_html(origStr,funCfg){try{const{loc:loc,idx:idx}=funCfg,idxInt=parseInt(idx),$=cheerio.load(origStr,_LsdString.__cheerioOptions);return _LsdString.__getElementBySelector($,loc,idxInt).html()}catch(err){return logerr(`##tmp cheerio c_html err: ${err}`),""}}static _c_length(origStr,funCfg){try{const{loc:loc}=funCfg,idxInt=-9999,$=cheerio.load(origStr,_LsdString.__cheerioOptions),$$=_LsdString.__getElementBySelector($,loc,idxInt);return String($$.length)}catch(err){return logerr(`##tmp cheerio c_length err: ${err}`),"0"}}static _c_text(origStr,funCfg){try{const{loc:loc,idx:idx}=funCfg,idxInt=parseInt(idx),$=cheerio.load(origStr,_LsdString.__cheerioOptions);return _LsdString.__getElementBySelector($,loc,idxInt).text()}catch(err){return logerr(`##tmp cheerio c_text err: ${err}`),""}}static _ceil(origStr){return String(Math.ceil(Number(origStr)))}static _closingsubstr(origStr,funCfg){const{openchar:openchar,startstr:startstr,position:position}=funCfg;if("string"!=typeof origStr\|\|"string"!=typeof openchar\|\|!["{","[","(","lt"].includes(openchar)\|\|"string"!=typeof startstr\|\|"number"!=typeof parseInt(position)\|\|parseInt(position)<0)return"";const openChar="lt"===openchar?"<":openchar,closingChar={"{":"}","[":"]","(":")","<":">"}[openChar];let openBracketCount=1,startPos=-1;if(startstr){if(startPos=origStr.indexOf(startstr),startPos<0)return""}else startPos=parseInt(position);const startIdx=origStr.indexOf(openChar,startPos);if(startIdx<0)return-1;let idx=startIdx+1;for(;idx<origStr.length;idx++){let char=origStr[idx];if(char===openChar)openBracketCount++;else if(char===closingChar&&(openBracketCount--,0===openBracketCount))break}const endIdx=idx<=origStr.length?idx+1:-1;return endIdx>startIdx?origStr.substring(startIdx,endIdx):""}static __getCompressOptions(funCfg,compressFlag){return funCfg\|\|compressFlag?void 0:{}}static _compress(origStr,funCfg){try{const{method:method,sourceencoding:sourceencoding,targetencoding:targetencoding,start:start,end:end}=funCfg,options=_LsdString.__getCompressOptions(funCfg,!0),origBuffer=Buffer.from(origStr,sourceencoding);let compressedBuffer=null;return"gzip"===method?compressedBuffer=zlib.gzipSync(origBuffer,options):"deflate"===method?compressedBuffer=zlib.deflateSync(origBuffer,options):"deflateraw"===method?compressedBuffer=zlib.deflateRawSync(origBuffer,options):"brotli"===method&&(compressedBuffer=zlib.brotliCompressSync(origBuffer,options)),compressedBuffer?_LsdString.__convertBufferToString(compressedBuffer,targetencoding,start,end):""}catch(err){return""}}static _decompress(origStr,funCfg){try{const{method:method,sourceencoding:sourceencoding,targetencoding:targetencoding,start:start,end:end}=funCfg,options=_LsdString.__getCompressOptions(funCfg,!1),origBuffer=Buffer.from(origStr,sourceencoding);let decompressedBuffer=Buffer.from(origStr,sourceencoding);return"gunzip"===method?decompressedBuffer=zlib.gunzipSync(origBuffer,options):"inflate"===method?decompressedBuffer=zlib.inflateSync(origBuffer,options):"inflateraw"===method?decompressedBuffer=zlib.inflateRawSync(origBuffer,options):"unzip"===method?decompressedBuffer=zlib.unzipSync(origBuffer,options):"brotli"===method&&(decompressedBuffer=zlib.brotliDecompressSync(origBuffer,options)),decompressedBuffer?_LsdString.__convertBufferToString(decompressedBuffer,targetencoding,start,end):""}catch(err){return""}}static _concat(origStr,funCfg){let str=origStr;const{str1:str1,str2:str2,str3:str3,str4:str4,str5:str5}=funCfg;return[str1,str2,str3,str4,str5].forEach(item=>{"string"==typeof item&&(str=str.concat(item))}),str}static _specialconvert(origStr,funCfg){let str=origStr;try{const{method:method}=funCfg;return"function"==typeof _LsdString.specialConvertFunObj[method]&&(str=_LsdString.specialConvertFunObj[method](origStr,funCfg)),str}catch(err){return str}}static _decode(origStr,funCfg){const{str1:str1,val1:val1,str2:str2,val2:val2,str3:str3,val3:val3,str4:str4,val4:val4,str5:str5,val5:val5,defaultval:defaultval}=funCfg;try{return origStr===str1?val1:origStr===str2?val2:origStr===str3?val3:origStr===str4?val4:origStr===str5?val5:defaultval}catch(err){return""}}static __convertBufferToString(buffer,encoding,startStr,endStr){const start=startStr?Number(startStr):0,end=endStr?Number(endStr):0;return buffer instanceof Buffer?end>0?buffer.toString(encoding,start,end):start>0?buffer.toString(encoding,start):buffer.toString(encoding):""}static _convertencoding(origStr,funCfg){const{sourceencoding:sourceencoding,targetencoding:targetencoding,start:start,end:end}=funCfg;try{const buffer=Buffer.from(origStr,sourceencoding);return _LsdString.__convertBufferToString(buffer,targetencoding,start,end)}catch(err){return""}}static _floor(origStr){return String(Math.floor(Number(origStr)))}static _hashcode(origStr,funCfg){const{hashmethod:hashmethod}=funCfg;return import_utils3.LsdUrl.getHashCode(origStr,hashmethod)}static _includes(origStr,funCfg){const{searchstr:searchstr}=funCfg;if("string"!=typeof searchstr)throw new Error("Invalid searchstr in _contains");return String(Number(origStr.includes(searchstr)))}static _insert(origStr,funCfg){const{str:str}=funCfg;if("string"!=typeof str)throw new Error("Invalid str in _insert");return str+origStr}static _itemstoobj(origStr,funCfg){const{split:itemSplit,kvsplit:kvSplit,keytrim:keyTrimFlag,keynows:keyNowsFlag,valuetrim:valueTrimFlag,valuenows:valueNowsFlag}=funCfg;if(!itemSplit\|\|!kvSplit)return"{}";try{let retObj={};const items=origStr.split(new RegExp(itemSplit)),reKvSplit=new RegExp(kvSplit);for(const item of items){const kv=item.split(reKvSplit);if(2!==kv.length)continue;let keyStr=keyTrimFlag?kv[0].trim():kv[0];keyNowsFlag&&(keyStr=keyStr.split("").filter(i=>i.trim()).join(""));let valueStr=valueTrimFlag?kv[1].trim():kv[1];valueNowsFlag&&(valueStr=valueStr.split("").filter(i=>i.trim()).join("")),keyStr&&(retObj[keyStr]=valueStr)}return JSON.stringify(retObj)}catch(err){return"{}"}}static _jsonparse(origStr,funCfg){const{key1:key1,key2:key2,key3:key3,key4:key4,key5:key5,key6:key6,keys:keys,length:length,srctype:srctype}=funCfg;let retObj=null;try{if("script"===srctype)return"";if(retObj=JSON.parse(origStr),"object"!=typeof retObj\|\|!retObj)return""}catch(err){return""}return retObj=(0,import_utils3.getMemberOfObject)(retObj,key1,key2,key3,key4,key5,key6),null==retObj?"":"string"==typeof retObj?retObj:"number"==typeof retObj?String(retObj):(keys&&(retObj=Array.from(Object.keys(retObj))),length&&Array.isArray(retObj)?String(retObj.length):JSON.stringify(retObj))}static _length(origStr){return String(origStr.length)}static _matchall(origStr,funCfg){try{const{pattern:pattern,flags:flags,itemvalue:itemvalue,filter:filter,start:start,end:end,resulttype:resulttype,join:join9}=funCfg,re=new RegExp(pattern,flags),matches=origStr.matchAll(re);if("boolean"===resulttype)return matches?"1":"0";if(!matches)return"";let strArray=[];for(const match of matches)if("match[0]"===itemvalue)strArray.push(match[0]);else{let item=itemvalue;for(let i=0;i<match.length;i++)item=item.replaceAll(`match[${i}]`,match[i]);match.index>=0&&(item=item.replaceAll("match.index",String(match.index))),strArray.push(item)}if(filter){const reFilter=new RegExp(filter);strArray=strArray.filter(item=>item.match(reFilter))}const startInt=parseInt(start),endInt=parseInt(end);return 0===startInt&&0===endInt\|\|(strArray=0===endInt?strArray.slice(startInt):strArray.slice(startInt,endInt)),"all"===resulttype?strArray.join(join9):"length"===resulttype?String(strArray.length):""}catch(err){return loginfo(err),""}}static _max(origStr,funCfg){try{const{split:split}=funCfg,nums=origStr.split(new RegExp(split)).filter(item=>item.trim()).map(item=>Number(item));return nums.length>0?String(nums.sort((a,b)=>a-b).pop()):""}catch(err){return loginfo(err),""}}static _min(origStr,funCfg){try{const{split:split}=funCfg,nums=origStr.split(new RegExp(split)).filter(item=>item.trim()).map(item=>Number(item));return nums.length>0?String(nums.sort((a,b)=>a-b)[0]):""}catch(err){return loginfo(err),""}}static _numeq(origStr,funCfg){const{num:num}=funCfg;return"string"!=typeof origStr\|\|"string"!=typeof num?"0":origStr.trim().match(/^\d+$/)&&num.trim().match(/^\d+$/)&&Number(origStr)===Number(num)?"1":"0"}static _numge(origStr,funCfg){const{num:num}=funCfg;return"string"!=typeof origStr\|\|"string"!=typeof num?"0":origStr.trim().match(/^\d+$/)&&num.trim().match(/^\d+$/)&&Number(origStr)>=Number(num)?"1":"0"}static _numgt(origStr,funCfg){const{num:num}=funCfg;return"string"!=typeof origStr\|\|"string"!=typeof num?"0":origStr.trim().match(/^\d+$/)&&num.trim().match(/^\d+$/)&&Number(origStr)>Number(num)?"1":"0"}static _numle(origStr,funCfg){const{num:num}=funCfg;return"string"!=typeof origStr\|\|"string"!=typeof num?"0":origStr.trim().match(/^\d+$/)&&num.trim().match(/^\d+$/)&&Number(origStr)<=Number(num)?"1":"0"}static _numlt(origStr,funCfg){const{num:num}=funCfg;return"string"!=typeof origStr\|\|"string"!=typeof num?"0":origStr.trim().match(/^\d+$/)&&num.trim().match(/^\d+$/)&&Number(origStr)<Number(num)?"1":"0"}static _notincludes(origStr,funCfg){const{searchstr:searchstr}=funCfg;if("string"!=typeof searchstr)throw new Error("Invalid str in _notcontains");return String(Number(!origStr.includes(searchstr)))}static _padEnd(origStr,funCfg){const{targetlen:targetlen,padstr:padstr}=funCfg,targetLen=parseInt(targetlen),padStr="string"==typeof padstr&&padstr.length>0?padstr:" ";return origStr.padEnd(targetLen,padStr)}static _padStart(origStr,funCfg){const{targetlen:targetlen,padstr:padstr}=funCfg,targetLen=parseInt(targetlen),padStr="string"==typeof padstr&&padstr.length>0?padstr:" ";return origStr.padStart(targetLen,padStr)}static _parseFloat(origStr){return String(parseFloat(origStr))}static _parseInt(origStr){return String(parseInt(origStr))}static _randomarryidxes(origStr,funCfg){const{join:join9}=funCfg;try{const length=parseInt(origStr);return[...Array(length).keys()].sort(()=>Math.random()-.5).join(join9)}catch(err){return""}}static _replace(origStr,funCfg){const{substr:substr,newstr:newstr,regexp:regexp,flags:flags}=funCfg;try{const regexpFlag=regexp;if("string"!=typeof substr)throw new Error("Invalid substr in _replace");if("string"!=typeof newstr)throw new Error("Invalid newstr in _replace");return regexpFlag?origStr.replace(RegExp(substr,flags),newstr):origStr.replace(substr,newstr)}catch(err){return origStr}}static _round(origStr){return String(Math.round(Number(origStr)))}static __sort(origArr,valtype,sorttype){let retArr;return retArr="string"===valtype?"asc"===sorttype?origArr.sort((a,b)=>a.localeCompare(b)):origArr.sort((a,b)=>b.localeCompare(a)):"asc"===sorttype?origArr.sort((a,b)=>parseFloat(a)-parseFloat(b)):origArr.sort((a,b)=>parseFloat(b)-parseFloat(a)),retArr}static _setcomplement(origStr,funCfg){const{array2:array2,split1:split1,split2:split2,sorttype:sorttype,valtype:valtype,limit:limit,join:join9}=funCfg;let reSplit1,reSplit2;try{reSplit1=new RegExp(split1),reSplit2=new RegExp(split2)}catch(err){throw new Error(`Invalid regexp split1: ${split1}, split2: ${split2} in _setcomplement`)}try{const limitInt=parseInt(limit);if(!["nosort","asc","desc"].includes(sorttype)\|\|!["string","number"].includes(valtype)\|\|limitInt<0\|\|"string"!=typeof array2\|\|"string"!=typeof join9)throw new Error(`Invalid limit: ${limit}, sorttype: ${sorttype}, or array2 in _setcomplement`);const strArr1=origStr.split(reSplit1),strArr2=array2.split(reSplit2);let resArr=(0,import_utils3.getComplementOfTwoArray)(strArr1,strArr2);return["asc","desc"].includes(sorttype)&&(resArr=_LsdString.__sort(resArr,valtype,sorttype)),limitInt>0?resArr.slice(0,limitInt).join(join9):resArr.join(join9)}catch(err){return loginfo(err),origStr}}static _setdifference(origStr,funCfg){const{array2:array2,split1:split1,split2:split2,sorttype:sorttype,valtype:valtype,limit:limit,join:join9}=funCfg;let reSplit1,reSplit2;try{reSplit1=new RegExp(split1),reSplit2=new RegExp(split2)}catch(err){throw new Error(`Invalid regexp split1: ${split1}, split2: ${split2} in _setcomplement`)}try{const limitInt=parseInt(limit);if(!["nosort","asc","desc"].includes(sorttype)\|\|!["string","number"].includes(valtype)\|\|limitInt<0\|\|"string"!=typeof array2\|\|"string"!=typeof join9)throw new Error(`Invalid limit: ${limit}, sorttype: ${sorttype}, or array2 in _setcomplement`);const strArr1=origStr.split(reSplit1),strArr2=array2.split(reSplit2);let resArr=(0,import_utils3.getDifferenceOfTwoArray)(strArr1,strArr2);return["asc","desc"].includes(sorttype)&&(resArr=_LsdString.__sort(resArr,valtype,sorttype)),limitInt>0?resArr.slice(0,limitInt).join(join9):resArr.join(join9)}catch(err){return loginfo(err),origStr}}static _setintersection(origStr,funCfg){const{array2:array2,split1:split1,split2:split2,sorttype:sorttype,valtype:valtype,limit:limit,join:join9}=funCfg;let reSplit1,reSplit2;try{reSplit1=new RegExp(split1),reSplit2=new RegExp(split2)}catch(err){throw new Error(`Invalid regexp split1: ${split1}, split2: ${split2} in _setcomplement`)}try{const limitInt=parseInt(limit);if(!["nosort","asc","desc"].includes(sorttype)\|\|!["string","number"].includes(valtype)\|\|limitInt<0\|\|"string"!=typeof array2\|\|"string"!=typeof join9)throw new Error(`Invalid limit: ${limit}, sorttype: ${sorttype}, or array2 in _setcomplement`);const strArr1=origStr.split(reSplit1),strArr2=array2.split(reSplit2);let resArr=(0,import_utils3.getIntersectionOfTwoArray)(strArr1,strArr2);return["asc","desc"].includes(sorttype)&&(resArr=_LsdString.__sort(resArr,valtype,sorttype)),limitInt>0?resArr.slice(0,limitInt).join(join9):resArr.join(join9)}catch(err){return loginfo(err),origStr}}static _setunion(origStr,funCfg){const{array2:array2,split1:split1,split2:split2,sorttype:sorttype,valtype:valtype,limit:limit,join:join9}=funCfg;let reSplit1,reSplit2;try{reSplit1=new RegExp(split1),reSplit2=new RegExp(split2)}catch(err){throw new Error(`Invalid regexp split1: ${split1}, split2: ${split2} in _setcomplement`)}try{const limitInt=parseInt(limit);if(!["nosort","asc","desc"].includes(sorttype)\|\|!["string","number"].includes(valtype)\|\|limitInt<0\|\|"string"!=typeof array2\|\|"string"!=typeof join9)throw new Error(`Invalid limit: ${limit}, sorttype: ${sorttype}, or array2 in _setcomplement`);const strArr1=origStr.split(reSplit1),strArr2=array2.split(reSplit2);let resArr=(0,import_utils3.getUnionOfTwoArray)(strArr1,strArr2);return["asc","desc"].includes(sorttype)&&(resArr=_LsdString.__sort(resArr,valtype,sorttype)),limitInt>0?resArr.slice(0,limitInt).join(join9):resArr.join(join9)}catch(err){return loginfo(err),origStr}}static _shuffle(origStr,funCfg){try{const{split:split}=funCfg;if("string"!=typeof split)throw new Error("Invalid split in _shuffle");const splitArr=origStr.split(new RegExp(split));return!Array.isArray(splitArr)\|\|splitArr.length<=1?origStr:splitArr.sort(()=>Math.random()-.5).join(split)}catch(err){return loginfo(err),origStr}}static _slice(origStr,funCfg){const{beginidx:beginidx,endidx:endidx}=funCfg,beginIdx=parseInt(beginidx),endIdx="string"==typeof endidx&&endidx.length>0?parseInt(endidx):void 0;return origStr.slice(beginIdx,endIdx)}static _sort(origStr,funCfg){try{const{split:split,valtype:valtype,sorttype:sorttype,join:join9}=funCfg;if("string"!=typeof split\|\|!["string","number"].includes(valtype)\|\|!["asc","desc"].includes(sorttype))throw new Error("Invalid split or valtype in _sort");const splitArr=origStr.split(new RegExp(split));return!Array.isArray(splitArr)\|\|splitArr.length<=1?origStr:"string"===valtype?"asc"===sorttype?splitArr.sort((a,b)=>a.localeCompare(b)).join(join9):splitArr.sort((a,b)=>b.localeCompare(a)).join(join9):"asc"===sorttype?splitArr.sort((a,b)=>parseFloat(a)-parseFloat(b)).join(join9):splitArr.sort((a,b)=>parseFloat(b)-parseFloat(a)).join(join9)}catch(err){return loginfo(err),""}}static _split(origStr,funCfg){try{const{split:split,limit:limit,filter:filter,start:start,end:end,resulttype:resulttype,val:val,insertstr:insertstr,join:join9}=funCfg;if("string"!=typeof split)throw new Error("Invalid split in _split");const reSplit=new RegExp(split),limitInt=parseInt(limit);let strArray=limitInt>0?origStr.split(reSplit,limitInt):origStr.split(reSplit);if(!Array.isArray(strArray))return"";if(filter){const reFilter=new RegExp(filter);strArray=strArray.filter(item=>item.match(reFilter))}const startInt=parseInt(start),endInt=parseInt(end);if(0===startInt&&0===endInt\|\|(strArray=0===endInt?strArray.slice(startInt):strArray.slice(startInt,endInt)),insertstr&&(strArray=strArray.map(item=>`${insertstr}${item}`)),"all"===resulttype)return strArray.join(join9);if("length"===resulttype)return String(strArray.length);if("index"===resulttype){const arrayIdx=strArray.findIndex(i=>i.trim()===val);return arrayIdx>=0?String(arrayIdx+1):""}return""}catch(err){return loginfo(err),""}}static _streq(origStr,funCfg){const{str:str}=funCfg;return"string"!=typeof str?"0":origStr===str?"1":"0"}static _substrAfter(origStr,funCfg){const{substr:substr,position:position,last:last,include:include}=funCfg,includeFlag=include,pos=parseInt(position);if("string"!=typeof substr)throw new Error("Invalid substr in _substrAfter");let index=0;return index=last?pos?origStr.lastIndexOf(substr,pos):origStr.lastIndexOf(substr):pos?origStr.indexOf(substr,pos):origStr.indexOf(substr),-1===index?origStr:String(includeFlag?origStr.substring(index):origStr.substring(index+substr.length))}static _substrBefore(origStr,funCfg){const{substr:substr,position:position,last:last,include:include}=funCfg,pos=parseInt(position),includeFlag=include;if("string"!=typeof substr)throw new Error("Invalid substr in substrBefore");let index=0;return index=last?pos?origStr.lastIndexOf(substr,pos):origStr.lastIndexOf(substr):pos?origStr.indexOf(substr,pos):origStr.indexOf(substr),-1===index?origStr:includeFlag?origStr.substring(0,index+substr.length):origStr.substring(0,index)}static _toBool(origStr){try{let strExp=origStr.replace(/true/g,"1");return strExp=strExp.replace(/false/g,"0"),strExp.search(/^[\s\d.+\-/()<>=!\|&:?]$/)>-1&&-1===strExp.search(/=>/)?String(Number(Boolean((0,eval)(strExp)))):(loginfo(`Invalid logic expression ${origStr} in _toBool`),!1)}catch(err){return!1}}static _toLowerCase(origStr){return origStr.toLowerCase()}static _toNum(origStr){let strExp=origStr;return strExp.search(/^[\s\d.+\-/()<>=!\|&:?]+$/)>-1&&-1===strExp.search(/=>/)&&strExp.trim().length>0?String(Number((0,eval)(strExp))):(loginfo(`Invalid arithmetic expression ${strExp} in _toNum`),"0")}static _toUpperCase(origStr){return origStr.toUpperCase()}static _trim(origStr){return origStr.trim()}static _trimEnd(origStr){return origStr.trimEnd()}static _trimStart(origStr){return origStr.trimStart()}static _uniq(origStr,funCfg){try{const{split:split,join:join9}=funCfg;if("string"!=typeof split\|\|"string"!=typeof join9)throw new Error("Invalid split in _uniq");const splitArr=origStr.split(new RegExp(split));return!Array.isArray(splitArr)\|\|splitArr.length<=1?origStr:[...new Set(splitArr)].join(join9)}catch(err){return loginfo(err),origStr}}static _urlencode(origStr,funCfg){try{const{method:method,encoding:encoding,percent:percent}=funCfg;if("URI"===method)return encodeURI(origStr);if("URIComponent"===method)return encodeURIComponent(origStr);if("iconv"===method){const str=iconv.encode(origStr,encoding).toString("hex").toUpperCase();if(!str\|\|str.length%2!=0)return logerr("Invalid str length in _urlencode"),"";if(percent){const matchResult=str.match(/../g);return matchResult?matchResult.map(item=>`%${item}`).join(""):""}return str}return logerr(`Invalid method ${method} in _urlencode`),""}catch(err){return logerr(err),""}}static _urldecode(origStr,funCfg){try{const{method:method,encoding:encoding,percent:percent}=funCfg;if("URI"===method)return decodeURI(origStr);if("URIComponent"===method)return decodeURIComponent(origStr);if("iconv"===method){const re=percent?/.../g:/../g,matchResult=origStr.match(re);if(!matchResult)return"";const hexChars=matchResult.map(item=>parseInt(item.slice(1),16));return iconv.decode(Buffer.from(hexChars),encoding)}return logerr(`Invalid method ${method} in _urlencode`),""}catch(err){return logerr(err),""}}static _urldelparams(origStr,funCfg){const{param1:param1,param2:param2,param3:param3}=funCfg;if("string"!=typeof param1)throw new Error("Invalid params in _urldel");let ret="";try{const url=new URL(origStr);if(!url\|\|!url.searchParams\|\|!url.searchParams.delete)return"";param1&&url.searchParams.delete(param1),param2&&url.searchParams.delete(param2),param3&&url.searchParams.delete(param3),ret=url.href}catch(err){ret=""}return ret\|\|""}static _urlget(origStr,funCfg){const{attrorpath:attrorpath,hashmethod:hashmethod,param:param,hostnameparts:hostnameparts,params:params,title:title,extname:extname,validextnames:validextnames}=funCfg;if("string"!=typeof attrorpath)throw new Error("Invalid attr in _urlget");let ret="";try{ret=["href","origin","protocol","username","password","host","search","hash","hostname","port","hrefwithoutsearch","param"].includes(attrorpath)?import_utils3.LsdUrl.getUrlAttribute(origStr,attrorpath,hostnameparts,param):import_utils3.LsdUrl.getFilePathFromUrl(origStr,attrorpath,hashmethod,hostnameparts,params,extname,validextnames,title)}catch(err){ret=""}return"string"==typeof ret?ret:""}static __setUrlParam(url,param,val,replaceFlag){if(!url\|\|!url.searchParams\|\|!url.searchParams.get\|\|"string"!=typeof param\|\|"string"!=typeof val)return!1;if(!param)return!0;if(!replaceFlag){if(null!==url.searchParams.get(param))return!0}return url.searchParams.set(param,val),!0}static _urlsetparams(origStr,funCfg){const{param1:param1,val1:val1,param2:param2,val2:val2,param3:param3,val3:val3,replace:replace}=funCfg,replaceFlag=replace;if("string"!=typeof param1)throw new Error("Invalid params in _urlset");let ret="";try{const url=new URL(origStr);_LsdString.__setUrlParam(url,param1,val1,replaceFlag),_LsdString.__setUrlParam(url,param2,val2,replaceFlag),_LsdString.__setUrlParam(url,param3,val3,replaceFlag),ret=url.href}catch(err){ret=""}return ret\|\|""}static _funPerformers={c_attr:_LsdString._c_attr,c_data:_LsdString._c_data,c_html:_LsdString._c_html,c_length:_LsdString._c_length,c_text:_LsdString._c_text,ceil:_LsdString._ceil,closingsubstr:_LsdString._closingsubstr,compress:_LsdString._compress,concat:_LsdString._concat,convertencoding:_LsdString._convertencoding,decode:_LsdString._decode,decompress:_LsdString._decompress,floor:_LsdString._floor,hashcode:_LsdString._hashcode,includes:_LsdString._includes,insert:_LsdString._insert,itemstoobj:_LsdString._itemstoobj,jsonparse:_LsdString._jsonparse,length:_LsdString._length,matchall:_LsdString._matchall,max:_LsdString._max,min:_LsdString._min,myfun:_LsdString.#myfun,numeq:_LsdString._numeq,numge:_LsdString._numge,numgt:_LsdString._numgt,numle:_LsdString._numle,numlt:_LsdString._numlt,notincludes:_LsdString._notincludes,padend:_LsdString._padEnd,padstart:_LsdString._padStart,parsefloat:_LsdString._parseFloat,parseint:_LsdString._parseInt,randomarryidxes:_LsdString._randomarryidxes,replace:_LsdString._replace,round:_LsdString._round,setcomplement:_LsdString._setcomplement,setdifference:_LsdString._setdifference,setintersection:_LsdString._setintersection,setunion:_LsdString._setunion,shuffle:_LsdString._shuffle,slice:_LsdString._slice,sort:_LsdString._sort,specialconvert:_LsdString._specialconvert,split:_LsdString._split,streq:_LsdString._streq,substrafter:_LsdString._substrAfter,substrbefore:_LsdString._substrBefore,tobool:_LsdString._toBool,tolowercase:_LsdString._toLowerCase,tonum:_LsdString._toNum,touppercase:_LsdString._toUpperCase,trim:_LsdString._trim,trimend:_LsdString._trimEnd,trimstart:_LsdString._trimStart,uniq:_LsdString._uniq,urldecode:_LsdString._urldecode,urldelparams:_LsdString._urldelparams,urlencode:_LsdString._urlencode,urlget:_LsdString._urlget,urlsetparams:_LsdString._urlsetparams}};function getTransformExpStrSync(origStr,funCfgs,defaultVal=""){return LsdString.transformStrSync(origStr,funCfgs,defaultVal)}async function getTransformExpStr(origStr,funCfgs,defaultVal="",myFunPerformers={}){return await LsdString.transformStr(origStr,funCfgs,defaultVal,myFunPerformers)}var fs2=__toESM(require("fs"),1),path2=__toESM(require("path"),1),import_header_generator=require("header-generator"),import_dateformat=__toESM(require("dateformat"),1),uuid2=__toESM(require("uuid"),1),import_utils4=require("@letsscrapedata/utils"),import_hpagent=require("hpagent"),download=__toESM(require("download"),1);async function lsdDownload(url,proxyUrl="",headers=null){const options={};if(proxyUrl){const proxyAgent=new import_hpagent.HttpsProxyAgent({keepAlive:!0,keepAliveMsecs:1e3,maxSockets:256,maxFreeSockets:256,scheduling:"lifo",proxy:proxyUrl});options.agent={https:proxyAgent}}let origContent;return headers&&(options.headers=headers),origContent=proxyUrl\|\|headers?await download.default(url,void 0,options):await download.default(url),origContent}var GetPara=class _GetPara{static getSpecialFunObj={};static setGetSpecialFunObj(getSpecialFunObj){return _GetPara.getSpecialFunObj=getSpecialFunObj,!0}static getOtherFunObj={};static setGetOtherFunObj(getOtherFunObj){return _GetPara.getOtherFunObj=getOtherFunObj,!0}static noNeedToReturnContent="";static async getPara(getCfg,otherInfo){if(!getCfg\|\|!getCfg.type\|\|!otherInfo)return"";const getType=getCfg.type;let content="";switch(getType){case"addedtaskparas":"function"==typeof _GetPara.getOtherFunObj.getAddedTaskParasFun&&(content=await _GetPara.getOtherFunObj.getAddedTaskParasFun(getCfg));break;case"datetime":content=_GetPara.getDatetime(getCfg);break;case"execedtaskparas":"function"==typeof _GetPara.getOtherFunObj.getExecedTaskParasFun&&(content=await _GetPara.getOtherFunObj.getExecedTaskParasFun(getCfg));break;case"file":content=await _GetPara.getFile(getCfg,otherInfo);break;case"httpheaders":content=await _GetPara.getHttpHeaders(getCfg,otherInfo);break;case"queuedcntwithparas":"function"==typeof _GetPara.getOtherFunObj.getQueuedCntWithParasCfgFun&&(content=await _GetPara.getOtherFunObj.getQueuedCntWithParasCfgFun(getCfg));break;case"queuedtaskparas":"function"==typeof _GetPara.getOtherFunObj.getQueuedTaskParasFun&&(content=await _GetPara.getOtherFunObj.getQueuedTaskParasFun(getCfg));break;case"random":content=_GetPara.getRandom(getCfg);break;case"response":content=await _GetPara.getResponse(getCfg,otherInfo);break;case"special":content=await _GetPara.getSpecial(getCfg,otherInfo);break;case"sysparas":content="please use ${sysParas.paraname}!!!";break;default:(0,import_utils4.unreachable)(getType)}return content}static async getParaInElement(getCfg,element,otherInfo){if(!getCfg\|\|!getCfg.type\|\|!otherInfo)return"";const getType=getCfg.type,paras=otherInfo.paras;let content="";if("screenshot"===getType)content=await _GetPara.getScreenshot(element,getCfg,paras);else(0,import_utils4.unreachable)(getType);return content}static async getParaInPage(getCfg,page,otherInfo){if(!getCfg\|\|!getCfg.type\|\|!otherInfo)return"";const getType=getCfg.type,paras=otherInfo.paras;let content="";switch(getType){case"content":content=await _GetPara.getContent(page);break;case"cookies":content="";break;case"mhtml":content=await _GetPara.getMhtml(page,getCfg,paras);break;case"pdf":content=await _GetPara.getPdf(page,getCfg,paras);break;case"screenshot":content=await _GetPara.getScreenshot(page,getCfg,paras);break;case"title":content=await _GetPara.getTitle(page,getCfg);break;case"window":content=await _GetPara.getWindow(page,getCfg);break;default:(0,import_utils4.unreachable)(getType)}return content}static async getContent(page){try{if(!page)return"";return await page.content()}catch(err){return""}}static async getCookies(page,getCfg){try{if(!page)return"";const{urls:urlsStr,domain:domain,name:name,path:path9,value:value}=getCfg;let content="";urlsStr.length>0&&urlsStr.split(";");let cookies=await page.cookies();return domain&&cookies.filter(c=>c.domain===domain),name&&cookies.filter(c=>c.name===name),path9&&cookies.filter(c=>c.path===path9),content=value?cookies.length>0?cookies[0].value:"":JSON.stringify(cookies),content}catch(err){return""}}static getDatetime(getCfg){const{format:format}=getCfg;return(0,import_dateformat.default)(new Date,format)}static async getFile(getCfg,otherInfo){const paras=otherInfo.paras;try{const{url:url,pathtype:pathtype,hashmethod:hashmethod,hostnameparts:hostnameparts,params:params,extname:extname,validextnames:validextnames,basedir:basedir,setvar:setvar,pathvarname:pathvarname,proxy:proxyFlag,headers:headersStr,referer:referer}=getCfg;let pathStr=getCfg.path.trim();if(pathStr)pathStr=path2.join(pathStr);else{let filename=import_utils4.LsdUrl.getFilePathFromUrl(url,pathtype,hashmethod,Number(hostnameparts),params,extname,validextnames);pathStr=path2.join(basedir,filename)}if(pathStr){const subDir=path2.dirname(pathStr);if(!await(0,import_utils4.fsCheckOrCreateSubdir)(appConfig.cfg.baseFileDir,subDir))throw new Error("failed to checkOrCreateFileDir");pathStr=path2.join(appConfig.cfg.baseFileDir,pathStr)}const proxyUrl=proxyFlag&&otherInfo.proxyUrl?otherInfo.proxyUrl:"";let headers=null;(headersStr\|\|referer)&&(headers=headersStr?JSON.parse(headersStr):{},referer&&(headers.referer=referer));const origContent=await lsdDownload(url,proxyUrl,headers);return fs2.writeFileSync(pathStr,origContent),pathvarname&&paras&&!appConfig.RESERVED_PARA_NAMES.includes(pathvarname)&&(paras[pathvarname]=pathStr),setvar?origContent.toString("utf8"):_GetPara.noNeedToReturnContent}catch(err){throw new Error("error in get_file")}}static _genHeaderMethod={genLatestAndroid:import_header_generator.PRESETS.MODERN_ANDROID,genLatestDesktop:import_header_generator.PRESETS.MODERN_DESKTOP,genLatestLinux:import_header_generator.PRESETS.MODERN_LINUX,genLatestLinuxChrome:import_header_generator.PRESETS.MODERN_LINUX_CHROME,genLatestLinuxFirefox:import_header_generator.PRESETS.MODERN_LINUX_FIREFOX,genLatestMacos:import_header_generator.PRESETS.MODERN_MACOS,genLatestMacosChrome:import_header_generator.PRESETS.MODERN_MACOS_CHROME,genLatestMacosFirefox:import_header_generator.PRESETS.MODERN_MACOS_FIREFOX,genLatestMobile:import_header_generator.PRESETS.MODERN_MOBILE,genLatestWindows:import_header_generator.PRESETS.MODERN_WINDOWS,genLatestWindowsChrome:import_header_generator.PRESETS.MODERN_WINDOWS_CHROME,genLatestWindowsFirefox:import_header_generator.PRESETS.MODERN_WINDOWS_FIREFOX};static hasSameOrigin(sourceUrl,targetUrl,sourceType=""){try{const targetU=new URL(targetUrl),sourceU=new URL(sourceUrl);return targetU.origin===sourceU.origin\|\|!!["origStateHeaders","updatedStateHeaders"].includes(sourceType)&&targetU.hostname.endsWith(sourceU.hostname)}catch(err){return!1}}static async getHttpHeaders(getCfg,otherInfo){const{headermethod:headerMethod,origheaders:origHeaders}=getCfg,{genHeaderOptions:genHeaderOptions,newHttpHeaders:newHttpHeaders}=otherInfo,interceptionData=otherInfo.interceptionData;let headers={},secureHeadersFlag=!1,targetUrl="";try{if(_GetPara._genHeaderMethod[headerMethod]){secureHeadersFlag=!0;headers=new import_header_generator.HeaderGenerator(_GetPara._genHeaderMethod[headerMethod]).getHeaders()}else if("genNewByHttpOptions"===headerMethod){secureHeadersFlag=!0;headers=new import_header_generator.HeaderGenerator(genHeaderOptions).getHeaders()}else if("origheaders"===headerMethod)origHeaders&&(headers=JSON.parse(origHeaders));else{if(!interceptionData)throw new Error("logic error: no interceptionData");"cachedRequestHeaders"===headerMethod?(headers=Object.assign({},interceptionData.cachedRequestHeaders),targetUrl=interceptionData.cachedRequestUrl,interceptionData.updatedRequestUrl=targetUrl,interceptionData.updatedRequestHeaders=headers):"cachedResponseHeaders"===headerMethod?(headers=Object.assign({},interceptionData.cachedResponseHeaders),targetUrl=interceptionData.cachedResponseUrl,interceptionData.updatedResponseUrl=targetUrl,interceptionData.updatedResponseHeaders=headers):"origStateHeaders"===headerMethod?(headers=Object.assign({},interceptionData.origStateHeaders),targetUrl=interceptionData.origStateUrl,interceptionData.updatedStateUrl=targetUrl,interceptionData.updatedStateHeaders=headers):"updatedRequestHeaders"===headerMethod?(targetUrl=interceptionData.updatedRequestUrl,headers=interceptionData.updatedRequestHeaders):"updatedResponseHeaders"===headerMethod?(targetUrl=interceptionData.updatedResponseUrl,headers=interceptionData.updatedResponseHeaders):"updatedStateHeaders"===headerMethod?(targetUrl=interceptionData.updatedStateUrl,headers=interceptionData.updatedStateHeaders):logerr(`##get invalid httpMethod ${headerMethod} in getHttpHeaders`)}if(newHttpHeaders)for(const newHttpHeader of Object.values(newHttpHeaders)){const{name:name,source:source,value:value,deleteFlag:deleteFlag}=newHttpHeader;if(deleteFlag)delete headers[name];else if("value"===source)headers[name]=value;else if(!secureHeadersFlag&&source!==headerMethod){let sourceHeaders={},sourceUrl="";if("cachedRequestHeaders"===source)sourceUrl=interceptionData.cachedRequestUrl,sourceHeaders=interceptionData.cachedRequestHeaders;else if("cachedResponseHeaders"===source)sourceUrl=interceptionData.cachedResponseUrl,sourceHeaders=interceptionData.cachedResponseHeaders;else{if("origStateHeaders"!==source){logerr(`Invalid soure ${source} in httpheader`);continue}sourceUrl=interceptionData.origStateUrl,sourceHeaders=interceptionData.origStateHeaders}"string"==typeof sourceHeaders[name]&&_GetPara.hasSameOrigin(sourceUrl,targetUrl,source)?headers[name]=sourceHeaders[name]:value&&(headers[name]=value)}}return secureHeadersFlag?JSON.stringify(headers):""}catch(err){return logerr(err),""}}static async getMhtml(page,getCfg,paras){try{if(!page)return"";let pathStr=getCfg.path.trim();const{pathtype:pathtype,hashmethod:hashmethod,hostnameparts:hostnameparts,basedir:basedir,pathvarname:pathvarname}=getCfg;if(pathStr)pathStr=path2.join(pathStr);else{const urlStr=page.url();let title=await page.title();pathStr=import_utils4.LsdUrl.getFilePathFromUrl(urlStr,pathtype,hashmethod,Number(hostnameparts),"","mhtml","",title),pathStr=path2.join(basedir,pathStr)}const{data:data}=await page.sendCDPMessage("Page.captureSnapshot",{format:"mhtml"});if(pathStr){const subDir=path2.dirname(pathStr);if(!await(0,import_utils4.fsCheckOrCreateSubdir)(appConfig.cfg.baseFileDir,subDir))throw new Error("failed to checkOrCreateMhtmlDir");const fullName=path2.join(appConfig.cfg.baseFileDir,pathStr);fs2.writeFileSync(fullName,data),pathvarname&&paras&&!appConfig.RESERVED_PARA_NAMES.includes(pathvarname)&&(paras[pathvarname]=pathStr)}return String(data)}catch(err){throw logdbg(err),new Error("error in get_mhtml")}}static _getScreenSize(origSize){return String(Number(origSize))===origSize?`${origSize}px`:origSize}static async getPdf(page,getCfg,paras){try{if(!page)return"";const{pathtype:pathtype,hashmethod:hashmethod,hostnameparts:hostnameparts,basedir:basedir,scale:scale,displayheaderfooter:displayheaderfooter,headertemplate:headertemplate,footertemplate:footertemplate,printbackground:printbackground,landscape:landscape,pageranges:pageranges,format:format,width:width,height:height,top:top,right:right,bottom:bottom,left:left,onepage:onepage,hmargin:hmargin,pathvarname:pathvarname,setvar:setvar}=getCfg;let options={},pathStr=getCfg.path.trim();if(pathStr)pathStr=path2.join(pathStr);else{const urlStr=page.url();let title=await page.title();pathStr=import_utils4.LsdUrl.getFilePathFromUrl(urlStr,pathtype,hashmethod,Number(hostnameparts),"","pdf","",title),pathStr=path2.join(basedir,pathStr)}if(pathStr){const subDir=path2.dirname(pathStr);if(!await(0,import_utils4.fsCheckOrCreateSubdir)(appConfig.cfg.baseFileDir,subDir))throw new Error("failed to checkOrCreatePdfDir");pathStr=path2.join(appConfig.cfg.baseFileDir,pathStr),options.path=pathStr}if("1"!==scale&&(options.scale=Number(scale)),displayheaderfooter&&(options.displayHeaderFooter=displayheaderfooter),headertemplate&&(options.headerTemplate=headertemplate),footertemplate&&(options.footerTemplate=footertemplate),printbackground&&(options.printBackground=printbackground),landscape&&(options.landscape=landscape),pageranges&&(options.pageRanges=pageranges),top\|\|right\|\|bottom\|\|left\|\|displayheaderfooter){const margin={};top?margin.top=_GetPara._getScreenSize(top):displayheaderfooter&&!headertemplate&&(margin.top="60px"),bottom?margin.bottom=_GetPara._getScreenSize(bottom):displayheaderfooter&&!footertemplate&&(margin.bottom="60px"),right&&(margin.right=_GetPara._getScreenSize(right)),left&&(margin.left=_GetPara._getScreenSize(left)),options.margin=margin}if(onepage){const pageWidth=await page.pageWidth();let pageHeight=await page.pageHeight();hmargin&&(pageHeight+=Number(hmargin)),options.width=`${pageWidth}px`,options.height=`${pageHeight}px`,logdbg(`--------- pageHeight：${pageHeight}px`)}else width\|\|height?(width&&(options.width=_GetPara._getScreenSize(width)),height&&(options.height=_GetPara._getScreenSize(height))):"Letter"!==format&&(options.format=format);const buffer=await page.pdf(options);return pathvarname&&paras&&!appConfig.RESERVED_PARA_NAMES.includes(pathvarname)&&(paras[pathvarname]=pathStr),setvar?buffer.toString("base64"):_GetPara.noNeedToReturnContent}catch(err){throw logdbg(err),new Error("error in get_pdf")}}static getRandom(getCfg){try{const{min:minStr,max:maxStr}=getCfg,min=parseInt(minStr),max=parseInt(maxStr);if(min>=max)return"0";const rand=Math.round(Math.random()(max-min))+min;return String(rand)}catch(err){return"0"}}static async getResponse(getCfg,otherInfo){try{const{requrl:reqUrl,idx:idx,length:length}=getCfg;let responses=otherInfo?.interceptionData?.responses;if(!Array.isArray(responses)\|\|responses.length<=0\|\|!responses.every(r=>r.requestUrl))return"";if(reqUrl){const re=new RegExp(reqUrl);responses=responses.filter(r=>r.requestUrl.match(re))}const responsesCnt=responses.length;if(responsesCnt<=0)return"";if(length)return String(responsesCnt);const idxInt=parseInt(idx);let responseData="";return idxInt>=0&&idxInt<responsesCnt?responseData=responses[idxInt]?.responseData:idxInt<0&&idxInt+responsesCnt>=0&&(responseData=responses.slice(idxInt)[0]?.responseData),"string"==typeof responseData?responseData:""}catch(err){throw new Error("error in get_response")}}static async getScreenshot(pageOrElement,getCfg,paras){let options={};try{if(!pageOrElement)return"";let{basedir:basedir,path:pathStr,extname:extname,quality:quality,fullpage:fullpage,x:x,y:y,width:width,height:height,omitbackground:omitbackground,pathvarname:pathvarname,setvar:setvar}=getCfg;if(pathStr=pathStr.trim(),pathStr)pathStr=path2.join(basedir,pathStr);else if(pageOrElement.title){const title=await pageOrElement.title(),filename=`${import_utils4.LsdUrl.convertToValidPathname(title)}.${extname}`;pathStr=path2.join(basedir,filename)}else{const filename=`${uuid2.v4()}.${extname}`;pathStr=path2.join(basedir,filename)}if(pathStr){const subDir=path2.dirname(pathStr);if(!await(0,import_utils4.fsCheckOrCreateSubdir)(appConfig.cfg.baseFileDir,subDir))throw new Error("failed to checkOrCreatePdfDir");pathStr=path2.join(appConfig.cfg.baseFileDir,pathStr),options.path=pathStr}"jpeg"===extname&&(options.type="jpeg");const qualityInt=parseInt(quality);"jpeg"===extname&&qualityInt>=0&&(options.quality=qualityInt),fullpage&&(options.fullPage=fullpage);const widthInt=parseInt(width),heightInt=parseInt(height);if(widthInt>0&&heightInt>0){const clip={x:parseInt(x),y:parseInt(y),width:widthInt,height:heightInt};options.clip=clip}omitbackground&&(options.omitBackground=omitbackground);const buffer=await pageOrElement.screenshot(options);return pathvarname&&paras&&!appConfig.RESERVED_PARA_NAMES.includes(pathvarname)&&(paras[pathvarname]=pathStr),setvar&&buffer?buffer.toString("base64"):_GetPara.noNeedToReturnContent}catch(err){throw new Error("error in get_screenshot")}}static async getSpecial(getCfg,otherInfo){let content="";const method=getCfg.method;return"function"==typeof _GetPara.getSpecialFunObj[method]&&(content=await _GetPara.getSpecialFunObj[method](getCfg,otherInfo)),content}static async getTitle(page,getCfg){try{if(!page)return"";const{aspath:pathFlag}=getCfg,title=await page.title();return pathFlag?import_utils4.LsdUrl.convertToValidPathname(title):title}catch(err){return""}}static async getWindow(page,getCfg){try{const{key1:key1,key2:key2,key3:key3,key4:key4,key5:key5,key6:key6,keys:keysFlag,length:lengthFlag}=getCfg;if(!page\|\|!key1)return"";const keys=[key1,key2,key3,key4,key5,key6];let str=await page.windowMember(keys);if(!keysFlag&&!lengthFlag)return str;const obj=JSON.parse(str);return obj&&"object"==typeof obj?keysFlag?Array.from(Object.keys(obj)).join(","):lengthFlag&&Array.isArray(obj)?String(obj.length):"":""}catch(err){return logwarn(err),""}}},path3=__toESM(require("path"),1),fsp=__toESM(require("fs/promises"),1),import_utils5=require("@letsscrapedata/utils");async function responseCacheCb(response,options,pageData){if(options&&response)try{const{execData:execData,contentType:contentType,tabname:tabname,cacheFlag:cacheFlag,requestHeadersFlag:requestHeadersFlag,responseHeadersFlag:responseHeadersFlag,interceptionData:interceptionData}=options,{pageUrl:pageUrl,cookies:cookies}=pageData,request=response.request(),requestMethod=request.method(),requestUrl=request.url();let data=request.postData();const requestData=data\|\|"",responseData="text"===contentType?await response.text():"";if(tabname&&"none"!==tabname&&(execData[tabname]\|\|(execData[tabname]=[]),execData[tabname].push({pageUrl:pageUrl,requestMethod:requestMethod,requestUrl:requestUrl,requestData:requestData,responseData:responseData,size:String(responseData.length)})),cacheFlag&&interceptionData.responses.push({pageUrl:pageUrl,requestMethod:requestMethod,requestUrl:requestUrl,requestData:requestData,responseData:responseData}),requestHeadersFlag){const requestHeaders=Object.assign({},request.allHeaders?await request.allHeaders():request.headers());!requestHeaders.cookie&&cookies&&(requestHeaders.cookie=cookies),interceptionData.cachedRequestUrl=requestUrl,interceptionData.cachedRequestHeaders=requestHeaders,logdbg("requestUrl: ",requestUrl),logdbg("requestHeaders: ",requestHeaders)}if(responseHeadersFlag){const responseHeaders=Object.assign({},response.allHeaders?await response.allHeaders():response.headers());interceptionData.cachedResponseUrl=requestUrl,interceptionData.cachedResponseHeaders=responseHeaders,logdbg("responseHeaders: ",responseHeaders)}}catch(err){logerr(err)}}var isHttpResponse=item=>void 0!==item.buffer;async function responseSaveCb(response,options,pageData){if(!options\|\|!response)return;const request=response.request(),requestUrl=request.url(),{execData:execData,tabname:tabname,pathType:pathType,hashMethod:hashMethod,baseDir:baseDir,hostnameParts:hostnameParts,params:params,extname:extname,validExtnames:validExtnames,encoding:encoding}=options,{pageUrl:pageUrl}=pageData;let filename=import_utils5.LsdUrl.getFilePathFromUrl(requestUrl,pathType,hashMethod,hostnameParts,params,extname,validExtnames);requestUrl.startsWith("http")\|\|logdbg("@@response: ",response),logdbg("@@reqUrl: ",requestUrl.slice(0,100)),logdbg("@@filename: ",filename);let pathStr=path3.join(baseDir,filename);if(!pathStr)return;const subDir=path3.dirname(pathStr);if(!await(0,import_utils5.fsCheckOrCreateSubdir)(appConfig.cfg.baseFileDir,subDir))throw new Error("failed to checkOrCreateReponseDir");const absolutePathStr=path3.join(appConfig.cfg.baseFileDir,pathStr),buffer=isHttpResponse(response)?await response.buffer():await response.body();let bufferEncoding=encoding;if(!encoding){const resourceType=request.resourceType();bufferEncoding=["document","stylesheet","script","xhr","fetch"].includes(resourceType)?"utf8":"binary"}if(await fsp.writeFile(absolutePathStr,buffer,bufferEncoding),tabname&&"none"!==tabname&&execData){const requestMethod=request.method();let data=request.postData();const record={pageUrl:pageUrl,requestMethod:requestMethod,requestUrl:requestUrl,requestData:data\|\|"",path:pathStr,size:String(buffer.length)};execData[tabname]?execData[tabname].push(record):execData[tabname]=[record]}}var import_node_fs=__toESM(require("fs"),1),import_utils7=require("@letsscrapedata/utils"),path4=__toESM(require("path"),1),fs3=__toESM(require("fs"),1),import_utils6=require("@letsscrapedata/utils"),solvers=[],addonFlag=!1;async function getScriptFromServer(scriptName){try{let content="",url=`${appConfig.cfg.urlPrefix}/script?name=${scriptName}`;const response=await fetch(url);if(200!==response.status)return"";const json=await response.json();return json&&0===json.code&&json.data&&"string"==typeof json.data?(content=Buffer.from(json.data,"base64").toString("utf8"),content):""}catch(err){return loginfo(err),""}}async function loadScriptIfNeeded(){if(addonFlag)return!0;addonFlag=!0;try{const addons=appConfig.cfg.lsdLaunchOptions.addons,addonPath=Array.isArray(addons)&&addons.length>0?addons[0]:"",captchas=appConfig.cfg.captcha.captchas;if(!addonPath\|\|!Array.isArray(captchas)\|\|0===captchas.length)return!0;let filename="";if(filename=path4.join(addonPath,"manifest.json"),!(0,import_utils6.isFileSync)(filename)){let manifestContent="ewogICJtYW5pZmVzdF92ZXJzaW9uIjogMywKICAibmFtZSI6ICJjYW1vdWZveCBhZGRvbiIsCiAgInZlcnNpb24iOiAiMS4wIiwKICAicGVybWlzc2lvbnMiOiBbCiAgICAic2NyaXB0aW5nIgogIF0sCiAgIndlYl9hY2Nlc3NpYmxlX3Jlc291cmNlcyI6IFsKICAgIHsKICAgICAgInJlc291cmNlcyI6IFsKICAgICAgICAic2NyaXB0cy8qIgogICAgICBdLAogICAgICAibWF0Y2hlcyI6IFsKICAgICAgICAiPGFsbF91cmxzPiIKICAgICAgXQogICAgfQogIF0sCiAgImNvbnRlbnRfc2NyaXB0cyI6IFsKICAgIHsKICAgICAgIm1hdGNoZXMiOiBbCiAgICAgICAgIjxhbGxfdXJscz4iCiAgICAgIF0sCiAgICAgICJqcyI6IFsKICAgICAgICAiaW5qZWN0LmpzIgogICAgICBdLAogICAgICAicnVuX2F0IjogImRvY3VtZW50X3N0YXJ0IgogICAgfQogIF0KfQ==";manifestContent=Buffer.from(manifestContent,"base64").toString("utf8"),fs3.writeFileSync(filename,manifestContent,"utf8")}if(filename=path4.join(addonPath,"inject.js"),!(0,import_utils6.isFileSync)(filename)){let injectContent="YXN5bmMgZnVuY3Rpb24gaW5qZWN0U2NyaXB0cygpIHsKICB0cnkgewogICAgZm9yIChjb25zdCBuYW1lIG9mIG5hbWVzKSB7CiAgICAgIHRyeSB7CiAgICAgICAgIGNvbnN0IHJlc3BvbnNlID0gYXdhaXQgZmV0Y2goY2hyb21lLnJ1bnRpbWUuZ2V0VVJMKGBzY3JpcHRzLyR7bmFtZX0uanNgKSk7CiAgICAgICAgY29uc3QgY29udGVudCA9IGF3YWl0IHJlc3BvbnNlLnRleHQoKTsKCiAgICAgICAgaWYgKGNvbnRlbnQpIHsKICAgICAgICAgIGNvbnN0IHNjcmlwdCA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoInNjcmlwdCIpOwogICAgICAgICAgc2NyaXB0LnRleHRDb250ZW50ID0gY29udGVudDsKICAgICAgICAgIGRvY3VtZW50LmRvY3VtZW50RWxlbWVudC5hcHBlbmRDaGlsZChzY3JpcHQpOwogICAgICAgICAgc2NyaXB0LnJlbW92ZSgpOwogICAgICAgIH0KICAgICAgfSBjYXRjaCAoZXJyKSB7CiAgICAgICAgY29uc29sZS5lcnJvcihgRmFpbGVkIHRvIGluamVjdCBzY3JpcHQgJHtuYW1lfTpgLCBlcnIpOwogICAgICB9CiAgICB9CiAgfSBjYXRjaCAoZXJyKSB7CiAgICBjb25zb2xlLmVycm9yKGVycik7CiAgfQp9CgppbmplY3RTY3JpcHRzKCk7Cg==";injectContent=Buffer.from(injectContent,"base64").toString("utf8");injectContent=`${`const names = ${JSON.stringify(captchas)};`}\n\n${injectContent}`,fs3.writeFileSync(filename,injectContent,"utf8")}if(filename=path4.join(addonPath,"scripts"),!await(0,import_utils6.fsCheckOrCreateDir)(filename,!0,!0))return logerr("Failed to create directory"),!1;for(const name of captchas){if(filename=path4.join(addonPath,"scripts",`${name}.js`),(0,import_utils6.isFileSync)(filename))continue;const content=await getScriptFromServer(name);content?fs3.writeFileSync(filename,content,"utf8"):loginfo(`Failed to get script of ${name}`)}return!0}catch(err){return loginfo(err),!1}}async function addCapatchaProvider(provider){return!(!provider\|\|"object"!=typeof provider\|\|"string"!=typeof provider?.name())&&(solvers.push(provider),await loadScriptIfNeeded(),!0)}function getSolver(captchaType){if(0===solvers.length)return null;for(const solver of solvers)if(solver.canSolveCaptcha(captchaType))return solver;return null}async function executeTask(task){const solver=getSolver(task.type);if(!solver)throw new Error(`##captcha no cpatha provider to solve ${task.type}`);let response;if(!solver[task.type])throw new Error("##captcha logic error");return response=await solver[task.type](task),response}var processors=new Map,captchaTypeAttr="data-captcha-type",widgetIdAttr="data-widget-id",captchaOptions={rootTag:"head",parentTag:"capwgts",captchaTag:"capwgt",solutionAttr:"value"};function checkCaptchaType(captchaWidget,captchaType){if(captchaWidget.captchaType!==captchaType)throw new Error(`##captcha invalid captchaType ${captchaWidget.captchaType}`);return!0}function isValidStringAttr(val){return"string"==typeof val&&!!val&&!["null","undefined"].includes(val)}async function getCaptchasOnce(page,msgFunc,options=captchaOptions){let captchaWidgets=[];if(msgFunc){const msgData={opType:"get"},res=await page.evaluate(async([funcName,data])=>{const func=window[funcName];return"function"==typeof func?func(data):""},[msgFunc,msgData]);return Array.isArray(res)?(captchaWidgets=res,captchaWidgets):(logerr(`##captcha no function window.${msgFunc}`),[])}{const captchaSelector=`${options.rootTag} > ${options.parentTag} > ${options.captchaTag}`,captchaElements=await page.findElements(captchaSelector);if(0===captchaElements.length)return captchaWidgets;for(const captchaElement of captchaElements){const captchaWidget=await captchaElement.dataset();captchaWidgets.push(captchaWidget)}return captchaWidgets}}async function getWidgets(processor,page,msgFunc,options=captchaOptions){let captchaWidgets=await getCaptchasOnce(page,msgFunc,options);if(captchaWidgets.length>0)return captchaWidgets;const{needToRefresh:needToRefresh,maxRefreshTimes:maxRefreshTimes}=processor;if(needToRefresh&&maxRefreshTimes)for(let i=0;i<maxRefreshTimes;i++){if(!await needToRefresh(page))break;if(await page.reload(),await(0,import_utils7.sleep)(5e3),captchaWidgets=await getCaptchasOnce(page,msgFunc,options),captchaWidgets.length>0)return captchaWidgets}else if(await(0,import_utils7.sleep)(5e3),captchaWidgets=await getCaptchasOnce(page,msgFunc,options),captchaWidgets.length>0)return captchaWidgets;return[]}async function sendSolution(page,msgFunc,captchaWidget,solution,options=captchaOptions){const{captchaType:captchaType,widgetId:widgetId}=captchaWidget;if(msgFunc){const msgData={opType:"execute",captchaType:captchaType,widgetId:widgetId,solution:solution},res=await page.evaluate(async([funcName,data])=>{const func=window[funcName];return"function"==typeof func?func(data):""},[msgFunc,msgData]);return"string"==typeof res?res:(logerr(`##captcha no function window.${msgFunc}`),"failed:noMsgFunc")}{const selector=`${options.rootTag} > ${options.parentTag} > ${options.captchaTag}[${captchaTypeAttr}="${captchaType}"][${widgetIdAttr}="${widgetId}"]`,captchaElement=await page.findElement(selector);if(captchaElement){return await captchaElement.setAttribute(options.solutionAttr,solution)?"ok":"failed"}return"failed:noElement"}}var amazonGetTask=function(captchaWidget){checkCaptchaType(captchaWidget,"amazon");const{captchaType:captchaType,pageurl:pageurl,sitekey:sitekey,iv:iv,context:context,challengeScript:challengeScript,captchaScript:captchaScript}=captchaWidget;return{type:captchaType,websiteURL:pageurl,websiteKey:sitekey,iv:iv,context:context,challengeScript:challengeScript,captchaScript:captchaScript}},amazonExecutionSolution=async function(page,msgFunc,captchaWidget,solution){const{existing_token:existing_token}=solution;return await sendSolution(page,msgFunc,captchaWidget,existing_token)},amazonProceesor={type:"amazon",findMethods:["intercept","hunt"],scriptName:"amazon",getTask:amazonGetTask,executeSolution:amazonExecutionSolution};processors.set("amazon",amazonProceesor);var geetestGetTask=function(captchaWidget){checkCaptchaType(captchaWidget,"geetest");const{captchaType:captchaType,version:version,url:url,gt:gt,challenge:challenge,apiServer:apiServer,captchaId:captchaId}=captchaWidget;if(!version\|\|!["v3","v4"].includes(version))throw new Error(`## captcha invalid version ${captchaWidget.version} in ${captchaWidget.captchaType}`);if(!url)throw new Error(`## captcha invalid url in ${captchaWidget.captchaType}`);if(!("v3"!==version\|\|gt&&challenge))throw new Error(`## captcha invalid gt or challenge in ${captchaWidget.captchaType} v3`);if("v4"===version&&!captchaId)throw new Error(`## captcha invalid captchaId in ${captchaWidget.captchaType} v4`);const task={type:captchaType,version:"v4"===version?4:3,websiteURL:url};return isValidStringAttr(gt)&&(task.gt=gt),isValidStringAttr(challenge)&&(task.challenge=challenge),isValidStringAttr(apiServer)&&(task.geetestApiServerSubdomain=apiServer),isValidStringAttr(captchaId)&&(task.captchaId=captchaId),task},geetestExecutionSolution=async function(page,msgFunc,captchaWidget,solution){checkCaptchaType(captchaWidget,"geetest");let solString="";if("v3"===captchaWidget.version){const{challenge:challenge,validate:validate,seccode:seccode}=solution,geetest_seccode=seccode\|\|`${validate}\|jordan`;if("string"!=typeof challenge\|\|!challenge)throw new Error(`##captcha invalid challenge in ${captchaWidget.captchaType}`);if("string"!=typeof validate\|\|!validate)throw new Error(`##captcha invalid validate in ${captchaWidget.captchaType}`);if("string"!=typeof seccode\|\|!seccode)throw new Error(`##captcha invalid seccode in ${captchaWidget.captchaType}`);solString=(0,import_utils7.encodeToBase64)({geetest_challenge:challenge,geetest_validate:validate,geetest_seccode:geetest_seccode})}else{if("v4"!==captchaWidget.version)return"failed:invalidVersion";{const{captcha_id:captcha_id,lot_number:lot_number,pass_token:pass_token,gen_time:gen_time,captcha_output:captcha_output}=solution;if("string"!=typeof captcha_id\|\|!captcha_id)throw new Error(`##captcha invalid captcha_id in ${captchaWidget.captchaType}`);if("string"!=typeof lot_number\|\|!lot_number)throw new Error(`##captcha invalid lot_number in ${captchaWidget.captchaType}`);if("string"!=typeof pass_token\|\|!pass_token)throw new Error(`##captcha invalid pass_token in ${captchaWidget.captchaType}`);if("string"!=typeof gen_time\|\|!gen_time)throw new Error(`##captcha invalid gen_time in ${captchaWidget.captchaType}`);if("string"!=typeof captcha_output\|\|!captcha_output)throw new Error(`##captcha invalid captcha_output in ${captchaWidget.captchaType}`);solString=(0,import_utils7.encodeToBase64)({captcha_id:captcha_id,lot_number:lot_number,pass_token:pass_token,gen_time:gen_time,captcha_output:captcha_output})}}return await sendSolution(page,msgFunc,captchaWidget,solString)},geetestProceesor={type:"geetest",findMethods:["intercept"],scriptName:"geetest",getTask:geetestGetTask,executeSolution:geetestExecutionSolution};processors.set("geetest",geetestProceesor);var recaptchaGetTask=function(captchaWidget,options){checkCaptchaType(captchaWidget,"recaptcha");const{captchaType:captchaType,version:version,url:url,sitekey:sitekey,action:action,s:s,enterprise:enterprise,invisible:invisible,userAgent:userAgent,cookies:cookies,apiDomain:apiDomain}=captchaWidget;if(!version\|\|!["v2","v3"].includes(version))throw new Error(`## captcha invalid version ${captchaWidget.version} in ${captchaWidget.captchaType}`);if(!url\|\|!sitekey)throw new Error(`## captcha invalid url or sitekey in ${captchaWidget.captchaType}`);let minScore=.3;"number"==typeof options?.minScore&&options.minScore>0&&options.minScore<1&&(minScore=options.minScore);const task={type:captchaType,version:"v3"===version?3:2,websiteURL:url,websiteKey:sitekey,isEnterprise:"true"===enterprise,isInvisible:"true"===invisible,minScore:minScore};return isValidStringAttr(action)&&(task.action=action),isValidStringAttr(s)&&(task.s=s),isValidStringAttr(userAgent)&&(task.userAgent=userAgent),isValidStringAttr(cookies)&&(task.cookies=cookies),isValidStringAttr(apiDomain)&&(task.apiDomain=apiDomain),task},recaptchaExecutionSolution=async function(page,msgFunc,captchaWidget,solution){checkCaptchaType(captchaWidget,"recaptcha");const{gRecaptchaResponse:gRecaptchaResponse}=solution;if("string"!=typeof gRecaptchaResponse\|\|!gRecaptchaResponse)throw new Error(`##captcha invalid gRecaptchaResponse in ${captchaWidget.captchaType}`);return await sendSolution(page,msgFunc,captchaWidget,gRecaptchaResponse)},recaptchaProceesor={type:"recaptcha",findMethods:["intercept","hunt"],scriptName:"recaptcha",getTask:recaptchaGetTask,executeSolution:recaptchaExecutionSolution};processors.set("recaptcha",recaptchaProceesor);var turnstileNeedToRefresh=async function(page){return 0===(await page.findElements("#cf-turnstile")).length},turnstileGetTask=function(captchaWidget){checkCaptchaType(captchaWidget,"turnstile");const{captchaType:captchaType,url:url,sitekey:sitekey,action:action,cData:cData,chlPageData:chlPageData}=captchaWidget,task={type:captchaType,websiteURL:url,websiteKey:sitekey};return isValidStringAttr(action)&&(task.action=action),isValidStringAttr(cData)&&(task.cData=cData),isValidStringAttr(chlPageData)&&(task.chlPageData=chlPageData),task},turnstileExecutionSolution=async function(page,msgFunc,captchaWidget,solution){checkCaptchaType(captchaWidget,"turnstile");const{token:token}=solution;if("string"!=typeof token\|\|!token)throw new Error(`##captcha invalid token in ${captchaWidget.captchaType}`);return await sendSolution(page,msgFunc,captchaWidget,token)},turnstileProceesor={type:"turnstile",findMethods:["intercept","hunt"],scriptName:"turnstile",getTask:turnstileGetTask,executeSolution:turnstileExecutionSolution,needToRefresh:turnstileNeedToRefresh,maxRefreshTimes:2};function getScriptFromLocal(scriptName){try{const filename=`${"./packages/scraper/tmp/lsd/inject"}/${scriptName}.js`;return import_node_fs.default.readFileSync(filename,"utf8")}catch(err){return""}}processors.set("turnstile",turnstileProceesor);var scriptMap=new Map;async function getScript(scriptName){let content=scriptMap.get(scriptName);if(content)return content;if(content=getScriptFromLocal(scriptName),content\|\|(content=await getScriptFromServer(scriptName)),"string"==typeof content&&content.length>20)return scriptMap.set(scriptName,content),content;throw new Error(`##captcha failed to get script of ${scriptName}`)}async function injectScripts(page,captchaTypes){let ret=!0;if("camoufox"===page.browserContext().browser().browserControllerType())return!0;if(!Array.isArray(captchaTypes)\|\|0===captchaTypes.length)return ret=!1,ret;const injectedScriptNames=[];for(const captchaType of captchaTypes){const processor=processors.get(captchaType);if(!processor)continue;if(!processor.findMethods.includes("intercept")\|\|!processor.scriptName)continue;if(injectedScriptNames.includes(processor.scriptName))continue;const scriptContent=await getScript(processor.scriptName);if(!scriptContent){ret=!1;break}injectedScriptNames.push(processor.scriptName);await page.addPreloadScript(scriptContent)\|\|(ret=!1)}return page.setCloseWhenFree(!0),ret}async function solveTokenCaptcha(page,captchaType,_proxy,options){const solveResult={code:0,captchaType:captchaType,taskResults:[]};loginfo(`######################### start to sovle ${captchaType} @${new Date}: `,options);let result="ok";try{const processor=processors.get(captchaType);if(!processor)return logerr(`##captcha cannot process ${captchaType}`),solveResult.code=1001,solveResult;const{getTask:getTask,executeSolution:executeSolution}=processor,captchaWidgets=await getWidgets(processor,page,"");if(0===captchaWidgets.length)return solveResult;const captchaWidget=captchaWidgets[0];captchaType!==captchaWidget.captchaType&&logerr(`##captcha different captchaType ${captchaType} ${captchaWidget.captchaType}`);const task=getTask(captchaWidget,options),{solution:solution,errorCode:errorCode,taskId:taskId,createTime:createTime,endTime:endTime}=await executeTask(task);if(solveResult.taskResults.push({solution:solution,errorCode:errorCode,taskId:taskId,elapse:endTime-createTime}),solution&&(result=await executeSolution(page,"",captchaWidget,solution),result.startsWith("ok")\|\|(solveResult.code=1003,solveResult.message=result),result)){const selector="button[type='submit']";await page.findElement(selector)}return loginfo(`######################### end to sovle ${captchaType} @${new Date}: `,solveResult),solveResult}catch(err){return logerr(err),solveResult.code=1010,solveResult.message=err.message,solveResult}}var import_node_fs2=__toESM(require("fs"),1),import_utils8=require("@letsscrapedata/utils"),processors2=new Map;async function getTextOfElement(page,loc,attr){if("pageUrl"===attr)return page.url();const element=await page.findElement(loc);return element?"textContent"===attr?await element.textContent():await element.attribute(attr):""}async function getBufferOfImage(page,loc,proxy,idx){const imgEle=await page.findElement(loc);if(!imgEle)return null;try{const src=await imgEle.attribute("src");if(src.toLowerCase().includes(".gif")){const pageUrl=page.url(),url=import_utils8.LsdUrl.getAbsoulteUrl(pageUrl,src),gifBuffer=await lsdDownload(url,proxy?.proxyUrl);return"number"==typeof idx&&import_node_fs2.default.writeFileSync(`./captchaImage${idx}.gif`,gifBuffer),gifBuffer}{"lazy"===await imgEle.attribute("loading")&&(await imgEle.scrollIntoView(),await(0,import_utils8.sleep)(2e3));const imgBuffer=await imgEle.screenshot();return"number"==typeof idx&&import_node_fs2.default.writeFileSync(`./captchaImage${idx}.png`,imgBuffer),imgBuffer}}catch(err){return logerr(err),null}}var textGetTask=async function(page,options,proxy=null){const{type:type,imageLocs:imageLocs,case:isCase,commentLoc:commentLoc,commentAttr:commentAttr,inputLoc:inputLoc,enter:enter,submitLoc:submitLoc,checkLoc:checkLoc,lang:lang,math:math,maxLength:maxLength,minLength:minLength,question:question,space:space,textType:textType}=options;if(!Array.isArray(imageLocs)\|\|0===imageLocs.length)throw new Error("invalidParas");if(!enter&&!submitLoc)throw new Error("invalidParas");if(!(!question\|\|commentLoc&&commentAttr))throw new Error("invalidParas");if(!checkLoc)throw new Error("invalidParas");const images=[];for(const[idx,imageLoc]of imageLocs.entries()){const buffer=await getBufferOfImage(page,imageLoc,proxy,idx);if(!buffer)throw new Error("noRequiredElement:imageLoc");images.push(buffer)}if(!await page.findElement(inputLoc))throw new Error("noRequiredElement:inputLoc");let comment="";if(commentLoc&&(comment=await getTextOfElement(page,commentLoc,commentAttr),!comment))throw new Error("noRequiredElement:commentLoc");if(submitLoc){if(!await page.findElement(submitLoc))throw new Error("noRequiredElement:submitLoc")}return{type:type,images:images,case:isCase,comment:comment,lang:lang,math:math,maxLength:maxLength,minLength:minLength,question:question,space:space,textType:textType,websiteUrl:page.url()}},textExecuteSolution=async function(page,options,solution){const{text:text}=solution,{inputLoc:inputLoc,enter:enter,submitLoc:submitLoc,checkLoc:checkLoc,checkAttr:checkAttr,failedStr:failedStr}=options,inputElement=await page.findElement(inputLoc);if(!inputElement)throw new Error("noRequiredElement");let succeeded=!1;if(enter){if(succeeded=await inputElement.input(text,{enter:enter}),!succeeded)return"failed:input"}else{const submitElement=await page.findElement(submitLoc);if(!submitElement)throw new Error("noRequiredElement");if(succeeded=await inputElement.input(text),!succeeded)return"failed:input";if(succeeded=await submitElement.click(),!succeeded)return"failed:submit"}await(0,import_utils8.sleep)(1e3);const checkElement=await page.findElement(checkLoc);if(!checkElement)return"ok";if(!checkAttr)return"failed:checkFailed-noCheckElement";const failedText="textContent"===checkAttr?await checkElement.textContent():await checkElement.attribute(checkAttr);return!failedStr&&failedStr.length>0?`failed:checkFailed-${failedText}`:failedText.includes(failedStr)?`failed:checkFailed-${failedStr}`:"ok"},textProcessor={type:"text",getTask:textGetTask,executeSolution:textExecuteSolution};processors2.set("text",textProcessor);var coordinateGetTask=async function(page,options,proxy=null){const{type:type,imageLocs:imageLocs,commentLoc:commentLoc,commentAttr:commentAttr,lang:lang,maxClicks:maxClicks,minClicks:minClicks}=options;if(!Array.isArray(imageLocs)\|\|0===imageLocs.length)throw new Error("invalidParas");if(minClicks<0\|\|maxClicks>0&&minClicks>=maxClicks)throw new Error("invalidParas");const images=[];for(const[idx,imageLoc]of imageLocs.entries()){const buffer=await getBufferOfImage(page,imageLoc,proxy,idx);if(!buffer)throw new Error("noRequiredElement:imageLoc");images.push(buffer)}let comment="";if(commentLoc&&(comment=await getTextOfElement(page,commentLoc,commentAttr),!comment))throw new Error("noRequiredElement:commentLoc");return{type:type,images:images,comment:comment,lang:lang,maxClicks:maxClicks,minClicks:minClicks}},coordinateExecuteSolution=async function(page,options,solution){const{coordinates:coordinates}=solution,{minClicks:minClicks,maxClicks:maxClicks,imageLocs:imageLocs,submitLoc:submitLoc,checkLoc:checkLoc,checkAttr:checkAttr,failedStr:failedStr}=options;if(!Array.isArray(coordinates)\|\|coordinates.length<minClicks\|\|maxClicks>0&&coordinates.length>maxClicks)return"failed:invalidSolution";let succeeded=!1;const imageElement=await page.findElement(imageLocs[0]);if(!imageElement)throw new Error("noRequiredElement");const boundingBox=await imageElement.boundingBox();if(!boundingBox)return"failed:click";const{x:startX,y:startY}=boundingBox;for(const{x:x,y:y}of coordinates){if(succeeded=await page.mouseClick(startX+x,startY+y),!succeeded)return"failed:click";await(0,import_utils8.sleep)(200)}if(await(0,import_utils8.sleep)(1e3),submitLoc){const submitElement=await page.findElement(submitLoc);if(!submitElement)throw new Error("noRequiredElement");if(succeeded=await submitElement.click(),!succeeded)return"failed:submit";await(0,import_utils8.sleep)(1e3)}const checkElement=await page.findElement(checkLoc);if(!checkElement)return"ok";if(!checkAttr)return"failed:checkFailed-noCheckElement";const failedText="textContent"===checkAttr?await checkElement.textContent():await checkElement.attribute(checkAttr);return!failedStr&&failedStr.length>0?`failed:checkFailed-${failedText}`:failedText.includes(failedStr)?`failed:checkFailed-${failedStr}`:"ok"},coordinateProcessor={type:"text",getTask:coordinateGetTask,executeSolution:coordinateExecuteSolution};async function solveRecognitionCaptcha(page,options,proxy,tryTimes=1){const captchaType=options.type;loginfo(`######################### start to sovle ${captchaType} @${new Date}: `,options);const solveResult={code:0,captchaType:captchaType,taskResults:[]};let result="";try{const processor=processors2.get(captchaType);if(!processor)return logerr(`##captcha cannot process ${captchaType}`),solveResult.code=1001,solveResult;const{getTask:getTask,executeSolution:executeSolution}=processor;for(let i=1;i<=tryTimes;i++){const task=await getTask(page,options,proxy),{solution:solution,errorCode:errorCode,taskId:taskId,createTime:createTime,endTime:endTime}=await executeTask(task),taskResult={solution:solution,errorCode:errorCode,taskId:taskId,elapse:endTime-createTime};if(solveResult.taskResults.push(taskResult),solution){if(result=await executeSolution(page,options,solution),result.startsWith("ok"))return solveResult;if(result.startsWith("failed:checkFailed")){taskResult.errorCode="ERROR_CHECK_FAILED";continue}taskResult.errorCode="ERROR_SOLUTION_FAILED";break}}return solveResult.code=1003,solveResult.message=result,solveResult}catch(err){logerr(err);const message=err.message;return"invalidParas"===message?solveResult.code=1001:message.startsWith("noRequiredElement")?solveResult.code=1004:solveResult.code=1010,solveResult.message=message,loginfo(`######################### end to sovle ${captchaType} @${new Date}: `,solveResult),solveResult}}processors2.set("coordinate",coordinateProcessor);var uuid3=__toESM(require("uuid"),1);function calcCapCredits(credits){return credits>0?credits>>appConfig.GEN_CREDITS_BITS:0}function calcGenCredits(credits){return credits>0?credits&appConfig.GEN_CREDITS_AND:0}var TaskScraper=class extends import_node_events.default{#template;#actionConfigs;#fontsConfig;_fontsConfig;#myfunScripts;#version;#templateId;_templateId;#taskId;_taskId;#captchaOptions;#loginOptions;#defaultElementSource;#splitStr;#skipActions;#extractColumns;#performOneTask;#taskContext;_taskContext;#paras;_paras;#parasStr;#interceptionData;#actionPerformers;#errObj;#lastUrl;constructor(templateConfig,inParas,taskNetworkContext,taskOptions={}){super();const{template:template,actionConfigs:actionConfigs,fontsConfig:fontsConfig,myfunScripts:myfunScripts}=templateConfig,{proxy:proxy=null,page:page=null,browserApiContext:browserApiContext=null,standaloneApiContext:standaloneApiContext=null}=taskNetworkContext,{performOneTask:performOneTask2=appConfig.performOneTask,taskId:taskId=0,skipActions:skipActions=[],extractColumns:extractColumns=[],version:version="1.0",splitStr:splitStr="::",authInfo:authInfo={}}=taskOptions,origStateData=taskOptions.stateData?taskOptions.stateData:{cookies:[],localStorage:[],headers:{},userData:{}},defaultElementSource=taskOptions.defaultElementSource?taskOptions.defaultElementSource:template.defaultElementSource;try{this.#template=template,this.#actionConfigs=actionConfigs,this.#fontsConfig=fontsConfig,this._fontsConfig=fontsConfig,this.#myfunScripts=myfunScripts,this.#version=version,this.#templateId=template.templateId,this._templateId=template.templateId,this.#taskId=taskId,this._taskId=taskId,this.#captchaOptions=taskOptions.captchaOptions?taskOptions.captchaOptions:{captchaTypes:[]},this.#loginOptions=taskOptions.loginOptions?taskOptions.loginOptions:{loginWhens:[],pageMatchInfos:[]},this.#defaultElementSource=defaultElementSource,this.#skipActions=skipActions,this.#extractColumns=extractColumns,this.#splitStr=splitStr,this.#performOneTask=performOneTask2,this.#taskContext={proxy:proxy,page:page,browserApiContext:browserApiContext,standaloneApiContext:standaloneApiContext,stateApiContext:browserApiContext\|\|standaloneApiContext,taskApiContext:null,popupPage:null,browserElesStack:[],cheerioPage:null,cheerioElesStack:[],loopsStack:[],continueFlag:!1,breakFlag:!1,origStateData:origStateData,newStateData:null,execData:{},subtasks:[],credits:0,subtaskResults:[]};const sysParas=new SysParas(this.#taskContext,this.#templateId,this.#taskId,this.#version);this.#paras={inParas:inParas,sysParas:sysParas,authInfo:authInfo,userData:Object.assign({},origStateData.userData),responses:[]},this.#parasStr=this.#getParasStr();const origStateUrl=origStateData.localStorage.length>0?origStateData.localStorage[0].origin:origStateData.cookies.length>0?`https://${origStateData.cookies[0].domain}`:"";this.#interceptionData={responses:[],cachedRequestUrl:"",cachedRequestHeaders:{},cachedResponseUrl:"",cachedResponseHeaders:{},updatedRequestUrl:"",updatedRequestHeaders:{},updatedResponseUrl:"",updatedResponseHeaders:{},origStateUrl:origStateUrl,origStateHeaders:origStateData.headers,updatedStateUrl:"",updatedStateHeaders:{}},this.#actionPerformers={api:this.#performApiAction,break:this.#performBreakAction,captcha:this.#performCaptchaAction,click:this.#performClickAction,continue:this.#performContinueAction,exit:this.#performExitAction,extract:this.#performExtractAction,goto:this.#performGotoAction,hover:this.#performHoverAction,ifelse:this.#performIfelseAction,input:this.#performInputAction,intercept:this.#performInterceptAction,loopdowhile:this.#performLoopdowhileAction,loopfor:this.#performLoopforAction,loopineles:this.#performLoopinelesAction,loopinstr:this.#performLoopinstrAction,misc:this.__performMiscAction,scroll:this.#performScrollAction,select:this.#performSelectAction,setvar:this.#performSetvarAction,subtask:this.#performSubtaskAction,wait:this.#performWaitAction},this.#errObj={normal:-1,parasinvalid:-2,accountinvalid:-41,accountlocked:-42,loginfailed:-43,loginneeded:-49,tologin:-40,accessforbidden:-61,cfginvalid:-69,captchafailed:-52,switchproxynow:-86,switchproxyfuture:-87,other:-98,logicerror:-98,tetimeout:-79},this.#lastUrl="",this._taskContext=this.#taskContext,this._paras=this.#paras,this.#fontsConfig&&logdbg(this.#fontsConfig)}catch(err){throw logerr("##te Exeception in ScraperTask.constructor"),new Error(err)}}#getElementSource(eleCfg){const origElementSource=this.#getCfgStrAttr(eleCfg,"source");return"default"===origElementSource?this.#defaultElementSource:"browser"===origElementSource?"browser":"cheerio"===origElementSource?"cheerio":void(0,import_utils9.unreachable)(origElementSource)}#setFailedCredits(credits){this.#taskContext.credits=credits,logwarn(new Error)}#getActionName(actionType){return`action${actionType[0].toUpperCase()}${actionType.slice(1)}`}#canResetPage(page,actionType){const{browserElesStack:browserElesStack,cheerioElesStack:cheerioElesStack}=this.#taskContext;return!(!page\|\|browserElesStack.length>0\|\|cheerioElesStack.length>0)\|\|(this.#procErrname("cfginvalid",this.#getActionName(actionType),"resetPage"),!1)}#procErrname(errName,scenarioName,reason){"ignore"!==errName&&logwarn(`##tmp procErrname eName ${errName} scenario ${scenarioName} reason ${reason} tId ${this.#templateId} kId ${this.#taskId} p ${this.#parasStr}`);let credits=-69;if("string"!=typeof errName\|\|!errName\|\|!scenarioName)throw this.#taskContext.credits=credits,new Error("##te invalid paras in _procErrname");if("ignore"===errName)return!0;throw void 0!==this.#errObj[errName]?(credits=this.#errObj[errName],this.#taskContext.credits=credits,new Error(`##te exit with errname ${errName} in ${scenarioName}`)):(this.#taskContext.credits=credits,new Error(`##te invalid errName ${errName} in ${scenarioName}`))}_procErrname(errName,scenarioName,reason){return this.#procErrname(errName,scenarioName,reason)}#getParasStr(){if(this.#paras.inParas&&"object"==typeof this.#paras.inParas){const allParaNames=Array.from(Object.keys(this.#paras.inParas)),paraNames=allParaNames.filter(n=>n.startsWith("para")),aparaNames=allParaNames.filter(n=>n.startsWith("apara")),rparaNames=allParaNames.filter(n=>n.startsWith("rpara"));let parasStr=paraNames.map(n=>this.#paras.inParas[n]).join(this.#splitStr);return(aparaNames.length>0\|\|rparaNames.length>0)&&(parasStr=`${parasStr}@@${aparaNames.map(n=>this.#paras.inParas[n]).join(this.#splitStr)}`,rparaNames.length>0&&(parasStr=`${parasStr}@@${rparaNames.map(n=>this.#paras.inParas[n]).join(this.#splitStr)}`)),parasStr}return""}#setCheerioPage(htmlOrXml,isHtml=!0){return this.#taskContext.cheerioPage?this.#taskContext.cheerioPage.load(htmlOrXml,isHtml):this.#taskContext.cheerioPage=new import_controller3.CheerioPage(htmlOrXml,isHtml),!0}#setNormalVariable(varName,varValue){if("string"!=typeof varName\|\|"string"!=typeof varValue\|\|appConfig.RESERVED_PARA_NAMES.includes(varName)?this.#procErrname("cfginvalid","setNormalVar","invalidParas"):"cheerioHtml"!==varName&&"cheerioXml"!==varName\|\|(this.#taskContext.cheerioElesStack.length>0&&this.#procErrname("cfginvalid","setNormalVar","cheerioElesStack"),this.#setCheerioPage(varValue,"cheerioHtml"===varName)),varName.startsWith("userData.")){const newVarName=varName.slice(9);0===newVarName.length&&this.#procErrname("cfginvalid","setNormalVar","invalidUserDataVarName"),this.#paras.userData[newVarName]=varValue}else if(varName.startsWith("inParas_")){const newVarName=varName.slice(8);0!==newVarName.length&&void 0!==this.#paras.inParas[newVarName]\|\|this.#procErrname("cfginvalid","setNormalVar","invalidInParasVarName"),this.#paras.inParas[newVarName]=varValue,this.#parasStr=this.#getParasStr()}else this.#paras[varName]=varValue;return(varName.startsWith("dbgdbg")\|\|varName.startsWith("cx")&&varName.match(/^cx\d$/))&&logwarn(`##dbg setvar ${varName}: ${varValue}`),!0}#validateUrl(url){if("string"!=typeof url)return!1;const lowerUrl=url.toLowerCase();return lowerUrl.startsWith("https://")\|\|lowerUrl.startsWith("http://")}#validateCfgType(cfg){if(cfg?.[appConfig.XML_ATTR_KEY]?.type)return!0;throw new Error("##te Invalid element type config(validate)")}#transTemplToStr(templStr,paras){if("string"!=typeof templStr)throw new Error(`##te Invalid templStr: ${templStr}`);if(templStr.indexOf("${")<0)return templStr;try{return function(pars){const names=Object.keys(pars),vals=Object.values(pars);return new Function(...names,`return \`${this}\`;`)(...vals)}.call(templStr,paras)}catch(error){return""}}#getCfgStrAttr(cfg,cfgAttrName,transTemplFlag=!0,mustFlag=!0){if(!cfg\|\|"string"!=typeof cfgAttrName)throw new Error(`##te Invalid config or attribute, attribute: ${cfgAttrName}`);const attrVal=cfg[appConfig.XML_ATTR_KEY][cfgAttrName];if(null===attrVal)return attrVal;if(void 0===attrVal){if(mustFlag)throw new Error(`##te Config has no attribute ${cfgAttrName}`);return""}return!transTemplFlag\|\|["type","name","version"].includes(cfgAttrName)?attrVal:this.#transTemplToStr(attrVal,this.#paras)}_getCfgStrAttr(cfg,cfgAttrName,transTemplFlag=!0,mustFlag=!0){return this.#getCfgStrAttr(cfg,cfgAttrName,transTemplFlag,mustFlag)}#getCfgBoolAttr(cfg,cfgAttrName){if(!cfg)throw new Error(`##te Invalid action config or attribute, attribute: ${cfgAttrName}`);const attrVal=cfg[appConfig.XML_ATTR_KEY][cfgAttrName];if("boolean"!=typeof attrVal)throw loginfo(`##te attrNames: ${JSON.stringify(cfg[appConfig.XML_ATTR_KEY])}, attrName: ${cfgAttrName}`),new Error(`##te Element has no bool attribute ${cfgAttrName} or it is undefined/null`);return attrVal}_getCfgBoolAttr(cfg,cfgAttrName){return this.#getCfgBoolAttr(cfg,cfgAttrName)}#getCfgTextContext(cfg,mustFlag=!1){if(!cfg)throw new Error("##te Invalid action config or attribute");const attrVal=cfg[appConfig.XML_CHAR_KEY];if("string"==typeof attrVal)return attrVal;if(mustFlag)throw new Error("##te Config has no textContext");return""}_getCfgTextContext(cfg,mustFlag=!1){return this.#getCfgTextContext(cfg,mustFlag)}#getCfgChildrenOf(parentCfg,cfgType,mustFlag=!1){try{const cKey=appConfig.XML_CHILDREN_KEY,aKey=appConfig.XML_ATTR_KEY;let ret=parentCfg[cKey]?parentCfg[cKey][cfgType]:null;if(Array.isArray(ret))return ret;if(void 0===ret){const retObj={};return"elecontent"===cfgType?(retObj[aKey]={type:"text",multieles:!0,trim:!0,join:"\n",line:!1,top:!1},[retObj]):"ocrcfg"===cfgType?(retObj[aKey]={type:"api",nlchars:"",seperator:!1,lang:"zh"},[retObj]):[]}if(mustFlag)throw new Error(`##te Invalid cfgType ${cfgType} children of parentCfg ${JSON.stringify(parentCfg)}`);return[]}catch(err){return logerr(err),[]}}#generateElementCfg(loc,source="default",absolute=!1){const cfg={};return cfg[appConfig.XML_ATTR_KEY]={loc:loc,source:source,absolute:absolute},cfg}_getCfgChildrenOf(parentCfg,cfgType,mustFlag=!1){return this.#getCfgChildrenOf(parentCfg,cfgType,mustFlag)}#getOptionalCfgChildOf(parentCfg,cfgType){const cfgChildren=this.#getCfgChildrenOf(parentCfg,cfgType);return cfgChildren.length>0?cfgChildren[0]:null}#getRequiredCfgChildOf(parentCfg,cfgType){const cfgChildren=this.#getCfgChildrenOf(parentCfg,cfgType);if(0===cfgChildren.length)throw new Error(`##te There is no ${cfgType} tag config`);return cfgChildren[0]}#getPageInfo(){if(this.#taskContext.page)return this.#taskContext.page.pageInfo();throw new Error("Cannot get pageInfo from null browser page")}#isHeadless(){if(this.#taskContext.page)return this.#taskContext.page.browserContext().browser().isHeadless();throw new Error("Cannot get headless from null browser page")}#getIntVal(mathExprStr,roundType="ceil"){if("string"!=typeof mathExprStr)throw new Error(`##te Inavlid math expr: ${mathExprStr}`);if(!["ceil","round","floor"].includes(roundType))throw new Error(`##te Inavlid round type: ${roundType}`);return Number(getTransformExpStrSync(mathExprStr,[{type:"tonum"},{type:roundType}]))}#getBoolVal(boolExprStr){if("string"!=typeof boolExprStr)throw new Error(`##te Inavlid bool expr: ${boolExprStr}`);return Boolean(Number(getTransformExpStrSync(boolExprStr,[{type:"tobool"}])))}async#processWait(waitMsStr){let waitMsInt=parseInt(waitMsStr);return waitMsInt>0&&await(0,import_utils9.sleep)(waitMsInt),!0}#addTaskCredits(actCfg,genFlag=!0,capFlag=!0,inGenCredits=1,inCapCredits=1){const gencreditsFlag=actCfg?this.#getCfgBoolAttr(actCfg,"gen"):genFlag,capcreditsFlag=actCfg?this.#getCfgBoolAttr(actCfg,"cap"):capFlag,genCredits=gencreditsFlag?inGenCredits:0,capCredits=this.#template.capName&&capcreditsFlag?inCapCredits:0;return loginfo(`##credit addTaskCredits: gen ${genCredits}, cap ${capCredits}`),this.#taskContext.credits+=(capCredits<<appConfig.GEN_CREDITS_BITS)+genCredits,!0}#getDataLoginCaptchaInfo(actCfg,actionType){if(!["click","goto","input","scroll","select"].includes(actionType))throw new Error(`logic error actionType ${actionType} cannot call this function`);const{popuppage:popupPageType,wait:waitStr,errname:errName,eurl:eurl,eloc:eloc,pn1:pn1,pv1:pv1,pn2:pn2,pv2:pv2}=this.#transformStrBooleanAttrsInCfg(actCfg,!1),loginFlag=this.#getCfgBoolAttr(actCfg,"login")&&!!this.#captchaOptions.pageMatchInfos,captchaFlag=this.#getCfgBoolAttr(actCfg,"captcha")&&!!this.#loginOptions.pageMatchInfos,dataPageMatchInfo=eurl\|\|eloc\|\|pn1\|\|pv1\|\|pn2\|\|pv2?{eurl:eurl,eloc:eloc,pn1:pn1,pv1:pv1,pn2:pn2,pv2:pv2}:null;if("ignored"===popupPageType\|\|!loginFlag&&!captchaFlag&&!dataPageMatchInfo)return null;const popupSubtaskFlag=this.#getCfgBoolAttr(actCfg,"popupsubtask");let waitMs=parseInt(waitStr);waitMs>0\|\|(waitMs=0);const popupFlag="ignored"!==popupPageType,popupPageFlags={captchaFlag:popupFlag&&captchaFlag&&!!this.#captchaOptions?.captchaWhens?.includes("popup"),loginFlag:popupFlag&&loginFlag&&!!this.#loginOptions?.loginWhens?.includes("popup"),taskFlag:!(popupFlag&&popupSubtaskFlag)};if(popupFlag&&!popupPageFlags.captchaFlag&&!popupPageFlags.loginFlag&&!dataPageMatchInfo)return this.#procErrname("cfginvalid",this.#getActionName(actionType),"invalidPopupCfg"),null;let currentFlag="ignored"===popupPageType;const currentPageFlags={captchaFlag:captchaFlag&&!!this.#captchaOptions?.captchaWhens?.includes("current"),loginFlag:loginFlag&&!!this.#loginOptions?.loginWhens?.includes("current"),taskFlag:currentFlag&&!!dataPageMatchInfo};if(currentFlag&&!currentPageFlags.captchaFlag&&!currentPageFlags.loginFlag&&currentPageFlags.taskFlag)return this.#procErrname("cfginvalid",this.#getActionName(actionType),"invalidCurrentCfg"),null;return{actionType:actionType,errName:errName,dataPageMatchInfo:dataPageMatchInfo,waitMs:waitMs,popupFlag:popupFlag,popupPageType:popupPageType,popupPageFlags:popupPageFlags,currentFlag:currentFlag,currentPageFlags:currentPageFlags,captchaPageMatchInfos:this.#captchaOptions.pageMatchInfos,loginPageMatchInfos:this.#loginOptions.pageMatchInfos}}async#getPageMatchInfo(page,pageMatchInfos,checkLocFlag=!0){const urlStr=page.url();if(!urlStr.trim())return null;const url=new URL(urlStr),href=url.href.toLowerCase();for(const pmi of pageMatchInfos){if(!pmi.eurl&&!pmi.pn1&&!pmi.pn2)return null;let urlFlag=!pmi.eurl;if(pmi.eurl){const urls=pmi.eurl.split(",");urlFlag=href.startsWith(urls[0])&&urls.slice(1).every(item=>href.includes(item))}const p1Flag=!pmi.pn1\|\|pmi.pn1&&url.searchParams.get(pmi.pn1)===pmi.pv1,p2Flag=!pmi.pn2\|\|pmi.pn2&&url.searchParams.get(pmi.pn2)===pmi.pv2;if(!urlFlag\|\|!p1Flag\|\|!p2Flag)continue;let locFlag=!0;if(pmi.eloc&&checkLocFlag)try{for(let i=0;i<30;i++){if((await page.findElements(pmi.eloc)).length>0){locFlag=!0;break}await(0,import_utils9.sleep)(2e3)}}catch(err){locFlag=!1}if(locFlag)return pmi}return null}async#processPageLoginCaptcha(page,dataLoginCatchaInfo,scenario){const{actionType:actionType,dataPageMatchInfo:dataPageMatchInfo,popupPageType:popupPageType,errName:errName}=dataLoginCatchaInfo,pageFlags="popup"===scenario?dataLoginCatchaInfo.popupPageFlags:dataLoginCatchaInfo.currentPageFlags,{captchaFlag:captchaFlag,loginFlag:loginFlag,taskFlag:taskFlag}=pageFlags;if(captchaFlag&&dataLoginCatchaInfo.captchaPageMatchInfos){if(await this.#getPageMatchInfo(page,dataLoginCatchaInfo.captchaPageMatchInfos))return!0}if(loginFlag&&dataLoginCatchaInfo.loginPageMatchInfos){if(await this.#getPageMatchInfo(page,dataLoginCatchaInfo.loginPageMatchInfos))return this.#procErrname("tologin",this.#getActionName(actionType),scenario),!0}if(dataPageMatchInfo){if(await this.#getPageMatchInfo(page,[dataPageMatchInfo]))return"popup"===scenario&&(taskFlag?(this.#canResetPage(page,actionType),this.#taskContext.page=page):this.#taskContext.popupPage=page),!0;if("required"===popupPageType)return this.#procErrname(errName,this.#getActionName(actionType),"noMatchedPopupPage"),!1}return!0}async processNewPageLoginCaptcha(dataLoginCaptchaInfo,pageOperateFun,pageOperateArgs){const{actionType:actionType,errName:errName,waitMs:waitMs,popupPageType:popupPageType}=dataLoginCaptchaInfo,popupTimeout=appConfig.CHECK_POPUP_PAGE_TIMEOUT1;setTimeout(()=>this.emit(appConfig.EVENT_TYPES.lsdScraperPopup,{code:1009,message:"timeout",data:null}),popupTimeout),await pageOperateFun(...pageOperateArgs),waitMs&&await(0,import_utils9.sleep)(waitMs);const[response]=await(0,import_node_events2.once)(this,appConfig.EVENT_TYPES.lsdScraperPopup);loginfo(`##tmp Taskscraper received popup event ${response.code}`);const popupPage=response&&0===response.code&&response.data?response.data:null;if(!popupPage){if("required"!==popupPageType)return!0;this.#procErrname(errName,this.#getActionName(actionType),"noPopupPage")}return await this.#processPageLoginCaptcha(popupPage,dataLoginCaptchaInfo,"popup"),!0}#checkBrowserElementSource(elementSource,actionName){"browser"===elementSource&&this.#taskContext.page\|\|this.#procErrname("cfginvalid",this.#getActionName(actionName),"invalidBrowserElementSource")}async#getApiContxt(context){if("browser"===context)return this.#taskContext.browserApiContext;if("standalone"===context)return this.#taskContext.standaloneApiContext;if("state"===context)return this.#taskContext.stateApiContext;if("task"===context){if(!this.#taskContext.taskApiContext){const options=this.#taskContext.proxy?{proxy:this.#taskContext.proxy}:{};this.#taskContext.taskApiContext=await import_controller3.controller.newApiContext(options)}return this.#taskContext.taskApiContext}return null}#getApiRequestHeaders(reqUrl,headersSource,headersStr,referer,requestCfgs){let headers={};const headerCfgs=requestCfgs.filter(requestCfg=>"header"===this.#getCfgStrAttr(requestCfg,"type"));try{if("headers"!==headersSource\|\|"headers"===headersSource&&headersStr\|\|referer\|\|headerCfgs.length>0){let sourceUrl="";const interceptionData=this.#interceptionData;if("headers"===headersSource&&headersStr?headers=JSON.parse(headersStr):"cachedRequestHeaders"===headersSource?(sourceUrl=interceptionData.cachedRequestUrl,headers=Object.assign({},interceptionData.cachedRequestHeaders)):"cachedResponseHeaders"===headersSource?(sourceUrl=interceptionData.cachedResponseUrl,headers=Object.assign({},interceptionData.cachedResponseHeaders)):"origStateHeaders"===headersSource?(sourceUrl=interceptionData.origStateUrl,headers=Object.assign({},interceptionData.origStateHeaders)):"updatedRequestHeaders"===headersSource?(sourceUrl=interceptionData.updatedRequestUrl,headers=Object.assign({},interceptionData.updatedRequestHeaders)):"updatedResponseHeaders"===headersSource?(sourceUrl=interceptionData.updatedResponseUrl,headers=Object.assign({},interceptionData.updatedResponseHeaders)):"updatedStateHeaders"===headersSource?(sourceUrl=interceptionData.updatedStateUrl,headers=Object.assign({},interceptionData.updatedStateHeaders)):this.#procErrname("cfginvalid","actionApi","invalidHeadersSource"),headersSource.endsWith("Headers")&&(sourceUrl\|\|0===Array.from(Object.keys(headers)).length)){GetPara.hasSameOrigin(sourceUrl,reqUrl,headersSource)\|\|(headers={},this.#procErrname("cfginvalid","actionApi","differentOrigin"))}referer&&(headers.referer=referer);for(const headerCfg of headerCfgs){const name=this.#getCfgStrAttr(headerCfg,"name"),value=this.#getCfgStrAttr(headerCfg,"value");headers[name]=value}return headers}return null}catch(err){return loginfo(err),this.#procErrname("cfginvalid","actionApi","garhException"),null}}#setContentLengthHeaderIfNeeded(headers,requestData){return headers&&headers["Content-Length"]?(headers["Content-Length"]=String(requestData.length),!0):!(!headers\|\|!headers["content-length"])&&(headers["content-length"]=String(requestData.length),!0)}#getDataFromDataStr(dataStr,dataType){if("string"===dataType)return dataStr;if("number"===dataType)return Number(dataStr);if("boolean"===dataType)return!(!dataStr\|\|"false"===dataStr.toLowerCase()\|\|"0"===dataStr);if("null"===dataType)return null;if("undefined"===dataType)return;if("object"!==dataType)throw this.#procErrname("cfginvalid","actionApi","notObject"),new Error("unreachable");const data=(0,import_utils9.getJsonParseOfStr)(dataStr);if(!data\|\|"object"!=typeof data)throw this.#procErrname("cfginvalid","actionApi","jsonParseData"),new Error("unreachable");return data}#getRequestData(dataStr,dataType,requestCfgs,errName){try{const dataCfgs=requestCfgs.filter(requestCfg=>"data"===this.#getCfgStrAttr(requestCfg,"type"));if(dataCfgs.length>0&&"object"!==dataType)return this.#procErrname("cfginvalid","actionApi","invalidDataType"),"";if("object"!==dataType)return this.#getDataFromDataStr(dataStr,dataType);const data=this.#getDataFromDataStr(dataStr,dataType);for(const dataCfg of dataCfgs){const{type:type,subkeys:subkeys,value:value,datatype:datatype}=this.#transformStrBooleanAttrsInCfg(dataCfg);if("property"!==type\|\|!subkeys)throw this.#procErrname("cfginvalid","actionApi","invalidSubkeys"),new Error("unreachable");let subKeys=this.#getSubKeys(subkeys);const property=subKeys.pop(),parent=0===subKeys.length?data:this.#getMemberOfObject(data,subKeys);if(!parent\|\|"object"!=typeof parent)throw this.#procErrname("cfginvalid","actionApi","invalidParent"),new Error("unreachable");const propValue=this.#getDataFromDataStr(value,datatype);parent[property]=propValue}return data}catch(err){return loginfo(err),void this.#procErrname(errName,"actionApi","grdExeception")}}#setRequestData(requestOptions,context,formFlag,data){switch(context){case"browser":case"state":case"task":case"standalone":case"default":formFlag?requestOptions.form=data:requestOptions.data=data;break;case"fetch":if(formFlag){if(!data\|\|"object"!=typeof data)throw this.#procErrname("cfginvalid","actionApi","fetch"),new Error("unreachable");const formData=new FormData;for(const key of Object.keys(data))formData.set(key,data[key]);requestOptions.data=formData}else"string"==typeof data?requestOptions.body=data:requestOptions.data=data;break;default:(0,import_utils9.unreachable)(context)}return!0}async#getResponseOfApiRequest(context,timeout,reqUrl,requestOptions,errName){const response={statusCode:200,text:""};try{switch(context){case"browser":case"state":case"task":case"standalone":const apiContext=await this.#getApiContxt(context);if(!apiContext)throw new Error("##te null apiContext");timeout>0&&(requestOptions.timeout=timeout);const apiResponse=await apiContext.fetch(reqUrl,requestOptions);response.statusCode=apiResponse.status,response.text=apiResponse.text;break;case"default":timeout>0&&(requestOptions.timeout={request:timeout});const defaultResponse=await(0,import_got_scraping.gotScraping)(reqUrl,requestOptions);response.statusCode=defaultResponse.statusCode,response.text=defaultResponse.body;break;case"fetch":const fetchResponse=await fetch(reqUrl,requestOptions);response.statusCode=fetchResponse.status,response.text=await fetchResponse.text();break;default:(0,import_utils9.unreachable)(context)}return response}catch(err){return loginfo(err),this.#procErrname(errName,"actionApi","groarException"),response.statusCode=600,response.text=err.message,response}}#processStatusCodeOfApiResponse(actCfg,statusCode){let responseCfgs=this.#getCfgChildrenOf(actCfg,"response",!1);for(const responseCfg of responseCfgs){const codes=this.#getCfgStrAttr(responseCfg,"codes");if(!codes)throw this.#procErrname("cfginvalid","actionApi","invalideCodes"),new Error("unreachable");if(codes.split(",").map(code=>Number(code)).includes(statusCode)){const errName=this.#getCfgStrAttr(responseCfg,"errname");this.#procErrname(errName,"actionApi","statusCode")}}return!0}#validateApiResponseText(text,valErrName,pattern,flags){const re=this.#getRegExp(pattern,flags);if(re&&!text.match(re))throw this.#procErrname(valErrName,"actionApi","apiResponseText"),new Error("unreachable");return!0}#getCachedApiResponseText(origPath){try{const filename=this.#getAbsolutePath(origPath);return fs6.readFileSync(filename,"utf8")}catch(err){return""}}#getAbsolutePath(origPath){return"uuidv4"===origPath&&(origPath=`files/tid_${this.#templateId}_${uuid3.v4()}.txt`),origPath=path5.join(appConfig.cfg.baseFileDir,origPath)}#cacheApiResponse(requestMethod,requestUrl,requestData,responseData){const page=this.#taskContext.page,responseItem={pageUrl:page?page.url():"",requestUrl:requestUrl,requestMethod:requestMethod,requestData:requestData,responseData:responseData};return this.#interceptionData.responses.push(responseItem),!0}#saveApiResponse(text,origPath,errName){try{const filename=this.#getAbsolutePath(origPath);return fs6.writeFileSync(filename,text),!0}catch(err){return this.#procErrname(errName,"actionApi","sarExeception"),!1}}async#performApiAction(actCfg){let statusCode=200,text="",requestData="";const apiCfg=this.#transformStrBooleanAttrsInCfg(actCfg),{method:method,url:url,encodeuri:encodeUriFlag,session:sessionFlag,headerssource:headersSource,headers:headersStr,referer:referer,proxy:proxyFlag,timeout:timeoutStr,data:dataStr,datatype:dataType,form:formFlag,path:filePath,diskcache:diskCacheFlag,cache:cacheFlag,tabname:tabname,varname:varName,errname:errName,valerrname:valErrName,pattern:pattern,flags:flags}=apiCfg,context=apiCfg.context,timeout=timeoutStr?Number(timeoutStr):0,serverFlag=url.startsWith("/"),reqUrl=encodeUriFlag?encodeURI(url):url,requestCfgs=this.#getCfgChildrenOf(actCfg,"request",!1);if(!["GET","DELETE","POST","PUT"].includes(method.toUpperCase()))return this.#procErrname("cfginvalid","actionApi","invalidMethod"),"";let fromDiskFlag=!1;if(diskCacheFlag&&(text=this.#getCachedApiResponseText(filePath),text&&(fromDiskFlag=!0)),!text)if(serverFlag){if("function"==typeof GetPara.getOtherFunObj.getDataFromServerFun){const options=timeout>0?{timeout:timeout}:void 0;text=await GetPara.getOtherFunObj.getDataFromServerFun(method,reqUrl,dataStr,options)}}else{const requestOptions={method:method.toUpperCase()},proxy=this.#taskContext.proxy;proxy&&(proxyFlag&&proxy.proxyUrl&&(requestOptions.proxyUrl=proxy.proxyUrl),sessionFlag&&proxy.proxyUrl&&"default"===context&&(requestOptions.sessionToken=proxy.proxyUrl));const headers=this.#getApiRequestHeaders(reqUrl,headersSource,headersStr,referer,requestCfgs);if(headers&&(requestOptions.headers=headers),dataStr){const data=this.#getRequestData(dataStr,dataType,requestCfgs,errName);void 0!==data&&(requestData=JSON.stringify(data),this.#setRequestData(requestOptions,context,formFlag,data),this.#setContentLengthHeaderIfNeeded(headers,data))}const response=await this.#getResponseOfApiRequest(context,timeout,reqUrl,requestOptions,errName);statusCode=response.statusCode,text=response.text,200!==statusCode&&this.#processStatusCodeOfApiResponse(actCfg,statusCode)}if("ignore"!==valErrName&&pattern&&this.#validateApiResponseText(text,valErrName,pattern,flags),cacheFlag&&this.#cacheApiResponse(method,reqUrl,requestData,text),tabname&&"none"!==tabname){const execData=this.#taskContext.execData;execData[tabname]\|\|(execData[tabname]=[]),execData[tabname].push({pageUrl:this.#paras.sysParas.url,requestMethod:method,requestUrl:url,requestData:requestData,responseData:text,size:String(text.length)})}return!fromDiskFlag&&filePath&&this.#saveApiResponse(text,filePath,errName),varName&&this.#setNormalVariable(varName,text),text}async#performBreakAction(actCfg){this.#validateCfgType(actCfg),this.#taskContext.loopsStack.length>0?this.#taskContext.breakFlag=!0:logerr("##te Error: break cannot be called outside of loop")}#getLocOfChildElement(parentCfg){const elementCfg=this.#getRequiredCfgChildOf(parentCfg,"element");return this.#getCfgStrAttr(elementCfg,"loc")}#getLocsOfChildElements(parentCfg,minCnt=1){const elementCfgs=this.#getCfgChildrenOf(parentCfg,"element");return elementCfgs.length<minCnt?(this.#procErrname("cfginvalid","childElements","elementCnt"),[]):elementCfgs.map(elementCfg=>this.#getCfgStrAttr(elementCfg,"loc"))}async#processTextCaptcha(page,captchaCfg,proxy,tryTimes){const imageCfg=this.#getRequiredCfgChildOf(captchaCfg,"image"),imageLocs=this.#getLocsOfChildElements(imageCfg),commentCfg=this.#getOptionalCfgChildOf(captchaCfg,"comment"),commentLoc=commentCfg?this.#getLocOfChildElement(commentCfg):"",commentAttr=commentCfg?this.#getCfgStrAttr(commentCfg,"attr"):"",inputCfg=this.#getRequiredCfgChildOf(captchaCfg,"input"),inputLoc=this.#getLocOfChildElement(inputCfg),enter=this.#getCfgBoolAttr(inputCfg,"enter"),submitCfg=enter?null:this.#getOptionalCfgChildOf(captchaCfg,"submit"),submitLoc=enter\|\|!submitCfg?"":this.#getLocOfChildElement(submitCfg);if(!enter&&!submitCfg)return this.#procErrname("cfginvalid","actionCaptcha","invalidTextCfg"),!1;const checkCfg=this.#getRequiredCfgChildOf(captchaCfg,"check"),checkLoc=this.#getLocOfChildElement(checkCfg),checkAttr=this.#getCfgStrAttr(checkCfg,"attr"),failedStr=this.#getCfgStrAttr(checkCfg,"failedstr"),origCfg=this.#transformStrBooleanAttrsInCfg(captchaCfg),{lang:lang,math:math,maxlen:maxlen,minlen:minlen,question:question,space:space,texttype:texttype}=origCfg,options={type:"text",imageLocs:imageLocs,commentLoc:commentLoc,commentAttr:commentAttr,inputLoc:inputLoc,enter:enter,submitLoc:submitLoc,checkLoc:checkLoc,checkAttr:checkAttr,failedStr:failedStr,case:origCfg.case,lang:lang,math:math,maxLength:parseInt(maxlen),minLength:parseInt(minlen),question:question,space:space,textType:texttype};return await solveRecognitionCaptcha(page,options,proxy,tryTimes)}async#processCoordinateCaptcha(page,captchaCfg,proxy,tryTimes){const imageCfg=this.#getRequiredCfgChildOf(captchaCfg,"image"),imageLocs=this.#getLocsOfChildElements(imageCfg),commentCfg=this.#getOptionalCfgChildOf(captchaCfg,"comment"),commentLoc=commentCfg?this.#getLocOfChildElement(commentCfg):"",commentAttr=commentCfg?this.#getCfgStrAttr(commentCfg,"attr"):"";let submitLoc="";const submitCfg=this.#getOptionalCfgChildOf(captchaCfg,"submit");if(submitCfg&&(submitLoc=this.#getLocOfChildElement(submitCfg),!submitLoc))return this.#procErrname("cfginvalid","actionCaptcha","invalidCoordinateCfg"),!1;const checkCfg=this.#getRequiredCfgChildOf(captchaCfg,"check"),checkLoc=this.#getLocOfChildElement(checkCfg),checkAttr=this.#getCfgStrAttr(checkCfg,"attr"),failedStr=this.#getCfgStrAttr(checkCfg,"failedstr"),origCfg=this.#transformStrBooleanAttrsInCfg(captchaCfg),{lang:lang,maxclicks:maxclicks,minclicks:minclicks}=origCfg,options={type:"coordinate",imageLocs:imageLocs,commentLoc:commentLoc,commentAttr:commentAttr,submitLoc:submitLoc,checkLoc:checkLoc,checkAttr:checkAttr,failedStr:failedStr,lang:lang,maxClicks:parseInt(maxclicks),minClicks:parseInt(minclicks)};return await solveRecognitionCaptcha(page,options,proxy,tryTimes)}async#performCaptchaAction(actCfg){const page=this.#taskContext.page;if(!page)return this.#procErrname("cfginvalid","actionCaptcha","noPage"),!1;const tryTimes=parseInt(this.#getCfgStrAttr(actCfg,"try")),debug=this.#getCfgBoolAttr(actCfg,"debug"),captchaCfg=this.#getRequiredCfgChildOf(actCfg,"captcha"),captchaType=this.#getCfgStrAttr(captchaCfg,"type"),errName=this.#getCfgStrAttr(actCfg,"errname"),proxy=this.#taskContext.proxy;let result;switch(captchaType){case"amazon":case"funcaptcha":case"geetest":case"keycaptcha":case"mtcaptcha":case"turnstile":result=await solveTokenCaptcha(page,captchaType,proxy,{debug:debug});break;case"recaptcha":const minScore=Number(this.#getCfgStrAttr(captchaCfg,"minscore"));result=await solveTokenCaptcha(page,captchaType,proxy,{minScore:minScore,debug:debug});break;case"text":case"grid":case"slider":case"rotation":result=await this.#processTextCaptcha(page,captchaCfg,proxy,tryTimes);break;case"coordinate":result=await this.#processCoordinateCaptcha(page,captchaCfg,proxy,tryTimes);break;default:(0,import_utils9.unreachable)(captchaType)}return debug&&logwarn("##te captcha result: ",result),0!==result?.code&&this.#procErrname(errName,"actionCaptcha","cannotSolveCaptcha"),!0}async#performClickAction(actCfg){const page=this.#taskContext.page;if(!page)return this.#procErrname("cfginvalid","actionClick","noPage"),!1;this.#validateCfgType(actCfg);const actType=this.#getCfgStrAttr(actCfg,"type"),clickType=this.#getCfgStrAttr(actCfg,"clicktype"),tryTimes=parseInt(this.#getCfgStrAttr(actCfg,"try")),popupPageType=this.#getCfgStrAttr(actCfg,"popuppage"),navigateFlag=this.#getCfgBoolAttr(actCfg,"navigate"),errName=this.#getCfgStrAttr(actCfg,"errname"),eleCfgs=this.#getCfgChildrenOf(actCfg,"element",!0);"click"!==clickType&&"evaluate"!==clickType&&(logerr(`##task invalid clicktype ${clickType} in performClickAction`),this.#procErrname("cfginvalid","actionClick","invalidClickType"));const{elementSource:elementSource,elements:elements}=await this.#processElementCfgs(eleCfgs,tryTimes);if(this.#checkBrowserElementSource(elementSource,"click"),0===elements.length)return loginfo(`## te ---- no element to click, actCfg: ${JSON.stringify(actCfg).slice(0,200)}`),this.#procErrname(errName,"actionClick","noElements"),!1;const element=elements[0];try{logdbg(`##te click text: ${await element.innerText()}`)}catch(err){logdbg(err)}page.browserContext().browser().isHeadless()\|\|(logdbg(`##te pageId${page.id()} bringToFront in click`),await page.bringToFront());const downloadpath=this.#getCfgStrAttr(actCfg,"downloadpath"),baseDownloadDir=appConfig.cfg.baseDownloadDir,fullDownloadPath=path5.join(baseDownloadDir,downloadpath);if(downloadpath&&baseDownloadDir){if(!0===page.browserContext().browser().isHeadless()&&(logerr('##te please set browserHeadlesses "true" to download files by clicking'),this.#procErrname("cfginvalid","actionClick","invalidBrowserHeadlesses")),await(0,import_utils9.fsCheckOrCreateSubdir)(baseDownloadDir,downloadpath,"")){const method="Browser.setDownloadBehavior";try{loginfo(`##browser response of CDP ${method}: `,await page.sendCDPMessage(method,{behavior:"allow",downloadPath:fullDownloadPath}))}catch(err){err.message?.includes("does not support")?(logerr(err.message),this.#procErrname("cfginvalid","actionClick","invalidBrowserControllerTypes")):(logerr(err),logwarn(`##browser failed to send CDP message ${method}`))}}else this.#procErrname("other","actionClick","invalidDownloadDir")}const dataLoginCaptchaInfo=this.#getDataLoginCaptchaInfo(actCfg,actType);if(dataLoginCaptchaInfo&&dataLoginCaptchaInfo.popupFlag?await this.processNewPageLoginCaptcha(dataLoginCaptchaInfo,async ele=>{await ele.click({clickType:clickType})},[element]):await element.click({clickType:clickType}),"ignored"===popupPageType&&navigateFlag)try{await(0,import_utils9.sleep)(5e3)}catch(err){}dataLoginCaptchaInfo&&dataLoginCaptchaInfo.currentFlag&&await this.#processPageLoginCaptcha(page,dataLoginCaptchaInfo,"current");const waitMsStr=this.#getCfgStrAttr(actCfg,"wait");await this.#processWait(waitMsStr);const filename=this.#getCfgStrAttr(actCfg,"filename");let fullFilename="";if(fullDownloadPath&&filename){fullFilename=path5.join(fullDownloadPath,filename);let i=0;const maxLoops=300;for(i=0;i<maxLoops;i++){if(await(0,import_utils9.fileExist)(fullFilename))break;await(0,import_utils9.sleep)(1e3)}i>=maxLoops&&(logwarn(`##te download kId ${this.#taskId}: failed to download file ${fullFilename}`),this.#procErrname("other","actionClick","downloadTimeout"))}const varName=this.#getCfgStrAttr(actCfg,"pathvarname");return varName&&this.#setNormalVariable(varName,fullFilename),this.#addTaskCredits(actCfg),!0}async#performContinueAction(actCfg){return this.#validateCfgType(actCfg),this.#taskContext.loopsStack.length>0?(this.#taskContext.continueFlag=!0,!0):(logerr("##te Error: continue cannot be called outside of loop"),!1)}async#performExitAction(actCfg){this.#validateCfgType(actCfg);const errName=this.#getCfgStrAttr(actCfg,"errname");if(!errName)return!1;const credits=void 0!==this.#errObj[errName]?this.#errObj[errName]:-69;throw this.#taskContext.credits=credits,new Error(`##te action_throw with errname ${errName}`)}#getAbsoluteUrl(origUrl){try{const page=this.#taskContext.page,pageUrl=page?page.url():this.#paras.pageUrl?this.#paras.pageUrl:"";if(!pageUrl)return logwarn("Please execute page.goto or set variable pageUrl first"),origUrl;const url=new URL(origUrl,pageUrl);return url?url.href:origUrl}catch(err){return origUrl}}#getRegExp(pattern,flags=""){if(!pattern)return null;try{return flags?new RegExp(pattern,flags):new RegExp(pattern)}catch(err){return this.#procErrname("cfginvalid","regExp","invalidRegExp"),null}}async#processElecontentCfg(elecontentCfg,elements){let contentType="text";elecontentCfg&&(contentType=this.#getCfgStrAttr(elecontentCfg,"type"));let message="step00",content="",multielesFlag=!0,trimFlag=!1,joinStr="\n",onelineFlag=!1,topFlag=!1,absoluteFlag=!1,arr=[];try{if(elements.length>0){let firstElement=elements[0];switch(contentType){case"text":message="step11",elecontentCfg&&(multielesFlag=this.#getCfgBoolAttr(elecontentCfg,"multieles"),trimFlag=this.#getCfgBoolAttr(elecontentCfg,"trim"),joinStr=this.#getCfgStrAttr(elecontentCfg,"join"),onelineFlag=this.#getCfgBoolAttr(elecontentCfg,"line"),topFlag=this.#getCfgBoolAttr(elecontentCfg,"top")),message="step12";for(let element of elements){if(content=await element.innerText(topFlag),trimFlag&&(content=content.trim()),!multielesFlag)break;arr.push(content)}message="step13",multielesFlag&&(content="stringify"===joinStr?JSON.stringify(arr):arr.join(joinStr)),onelineFlag&&(content=content.replace(/[\s\r\n]+/g," ")),message="step14";break;case"attr":message="step21";const attrName=this.#getCfgStrAttr(elecontentCfg,"attrname"),attrName2=this.#getCfgStrAttr(elecontentCfg,"attrname2"),attrName3=this.#getCfgStrAttr(elecontentCfg,"attrname3"),attrName4=this.#getCfgStrAttr(elecontentCfg,"attrname4");trimFlag=this.#getCfgBoolAttr(elecontentCfg,"trim");const pattern=this.#getCfgStrAttr(elecontentCfg,"pattern"),boolAttrFlag=this.#getCfgBoolAttr(elecontentCfg,"boolattr");multielesFlag=this.#getCfgBoolAttr(elecontentCfg,"multieles"),joinStr=this.#getCfgStrAttr(elecontentCfg,"join"),onelineFlag=this.#getCfgBoolAttr(elecontentCfg,"line"),absoluteFlag=this.#getCfgBoolAttr(elecontentCfg,"absolute"),message="step22";for(let element of elements){if(boolAttrFlag)message="step22",content=await element.hasAttribute(attrName)?"1":"0";else{message="step23";const re=this.#getRegExp(pattern);for(const aName of[attrName,attrName2,attrName3,attrName4]){if(!aName)break;let attrVal=await element.attribute(aName);if(content="string"==typeof attrVal?attrVal:null===attrVal?"":String(attrVal),trimFlag&&(content=content.trim()),!re)break;if(content.match(re))break;content=""}absoluteFlag&&(content=this.#getAbsoluteUrl(content))}if(!multielesFlag)break;arr.push(content),message="step24"}multielesFlag&&(content="stringify"===joinStr?JSON.stringify(arr):arr.join(joinStr)),onelineFlag&&(content=content.replace(/[\s\r\n]+/g," ")),message="step25";break;case"innerhtml":content=await firstElement.innerHtml();break;case"length":content=String(elements.length);break;case"outerhtml":content=await firstElement.outerHtml();break;case"pictext":content="pictext";break;case"image":content="image";break;case"query":content="query";break;default:throw new Error(`##te Invalid content type: ${contentType}`)}}else"length"===contentType&&(content="0");return content}catch(err){throw loginfo(`##te processBrowserElecontentCfg: tId${this.#templateId} ecCfg: ${JSON.stringify(elecontentCfg).slice(0,200)}`),loginfo(`##te err: ${JSON.stringify(err)}`),loginfo(`##te contentType ${contentType}, content ${content}, onelineFlag ${onelineFlag}`),loginfo(`##te message ${message}`),loginfo(`##te $browserEles ${elements}, length ${elements.length}, e0 ${elements[0]}`),new Error(err)}}async#processTransformCfg(transformCfg,origContent){try{let retStr="";const defaulVal=this.#getCfgStrAttr(transformCfg,"defaultval"),usevarFlag=this.#getCfgBoolAttr(transformCfg,"usevar");let funCfgs=this.#getCfgChildrenOf(transformCfg,"fun",!0).map(cfg=>cfg[appConfig.XML_ATTR_KEY]);if(usevarFlag){const origFunCfgs=funCfgs;funCfgs=[];for(const origFunCfg of origFunCfgs){const funCfg=Object.assign({},origFunCfg);for(const key of Object.keys(funCfg))"string"==typeof funCfg[key]&&funCfg[key].includes("${")&&(funCfg[key]=this.#transTemplToStr(funCfg[key],this.#paras));funCfgs.push(funCfg)}}return retStr=await getTransformExpStr(origContent,funCfgs,defaulVal,this.#myfunScripts),retStr}catch(err){throw loginfo(`##te processTransformCfg: tId${this.#templateId} tCfg: ${JSON.stringify(transformCfg).slice(0,200)}`),loginfo(`##te err: ${JSON.stringify(err)}`),new Error(err)}}async#getSubtaskPage(popupFlag){if(popupFlag){if(!this.#taskContext.popupPage)throw new Error("No valid popup page");return this.#taskContext.popupPage}if(this.#taskContext.page){const page=await this.#taskContext.page.browserContext().getPage(!0);if(page)return page;throw new Error("No new page to perform sync subtask")}throw new Error("logic error in #getSubtaskPage")}async#freeSubtaskPage(page){return await page.free(),!0}#getSubtaskNetworkContext(page){const{proxy:proxy,browserApiContext:browserApiContext,standaloneApiContext:standaloneApiContext}=this.#taskContext;return{proxy:proxy,page:page,browserApiContext:browserApiContext,standaloneApiContext:standaloneApiContext}}async#processSyncSubtask(subtaskCfg){try{let content="";const templateId=parseInt(this.#getCfgStrAttr(subtaskCfg,"tid")),minLen=parseInt(this.#getCfgStrAttr(subtaskCfg,"minlen")),parasStr=this.#getCfgStrAttr(subtaskCfg,"parasstr");let errName=this.#getCfgStrAttr(subtaskCfg,"errname");const popuppageFlag=this.#getCfgBoolAttr(subtaskCfg,"popuppage");if(popuppageFlag&&!this.#taskContext.popupPage){const nopageErrName="ignore"===errName?"cfginvalid":errName;this.#procErrname(nopageErrName,"syncSubtask","noPopupPage")}if(parasStr.length>=minLen){const page=await this.#getSubtaskPage(popuppageFlag),{taskData:taskData}=await this.#performOneTask(templateId,parasStr,this.#getSubtaskNetworkContext(page),"memSync");await this.#freeSubtaskPage(page);const{credits:credits,execData:execData}=taskData;if(popuppageFlag&&(this.#taskContext.popupPage=null),logdbg(`##te memsyncsubtask t${templateId} p${parasStr} c${credits} data: ${JSON.stringify(execData)}`),credits>=0){if(execData&&1===Object.keys(execData).length&&Array.isArray(Object.values(execData)[0])){const tabData=Object.values(execData)[0];if(Array.isArray(tabData)&&1===tabData.length&&1===Object.keys(tabData[0]).length){const attrName=Object.keys(tabData[0])[0];content=tabData[0][attrName]}else content=JSON.stringify(execData)}else content=JSON.stringify(execData);let genCredits=calcGenCredits(credits),capCredits=calcCapCredits(credits);this.#addTaskCredits(null,!0,!0,genCredits,capCredits)}}else this.#procErrname(errName,"syncSubtask","failed");return content}catch(err){throw loginfo(`##te processSyncSubtask: tId${this.#templateId} pCfg: ${JSON.stringify(subtaskCfg).slice(0,200)}`),loginfo(`##te err: ${JSON.stringify(err)}`),new Error(err)}}#getGenHttpHeaderOptions(getCfg,origGetCfg){const genHeaderOptions={browserListQuery:getCfg.browserlist?getCfg.browerlist:"",browsers:["chrome"],operatingSystems:["windows"],devices:["desktop"],httpVersion:Number(getCfg.httpversion)>0?getCfg.httpversion:"2"},browserCfgs=this.#getCfgChildrenOf(origGetCfg,"httpbrowser");if(browserCfgs.length>0){const browserOptions=[];browserCfgs.forEach(browserCfg=>{const name=this.#getCfgStrAttr(browserCfg,"name");name\|\|this.#procErrname("cfginvalid","actGetHttpHeaders","invalidName");const browserOption={name:name},minVersion=parseInt(this.#getCfgStrAttr(browserCfg,"minversion"));minVersion>0&&(browserOption.minVersion=minVersion);const maxVersion=parseInt(this.#getCfgStrAttr(browserCfg,"maxversion"));maxVersion>0&&(browserOption.maxVersion=maxVersion);const httpVersion=this.#getCfgStrAttr(browserCfg,"httpversion");httpVersion&&Number(httpVersion)>0&&(browserOption.httpVersion=httpVersion),browserOptions.push(browserOption)}),genHeaderOptions.browsers=browserOptions}const osCfgs=this.#getCfgChildrenOf(origGetCfg,"httpos");if(osCfgs.length>0){const osOptions=[];osCfgs.forEach(osCfg=>{const osName=this.#getCfgStrAttr(osCfg,"os");osName\|\|this.#procErrname("cfginvalid","actGetHttpHeaders","invalidOsName"),osOptions.push(osName)}),genHeaderOptions.operatingSystems=osOptions}const deviceCfgs=this.#getCfgChildrenOf(origGetCfg,"httpdevice");if(deviceCfgs.length>0){const deviceOptions=[];deviceCfgs.forEach(deviceCfg=>{const deviceName=this.#getCfgStrAttr(deviceCfg,"device");deviceName\|\|this.#procErrname("cfginvalid","actGetHttpHeaders","invalidDeviceName"),deviceOptions.push(deviceName)}),genHeaderOptions.devices=deviceOptions}return genHeaderOptions}async#getEleOrTemplTransform(contentSrcType,parentCfg,objItem){try{let content="",selectedElements=[];if("element"===contentSrcType){const parentType=this.#getCfgStrAttr(parentCfg,"type"),eleCfgs=this.#getCfgChildrenOf(parentCfg,"element",!0),elecontentCfg=this.#getRequiredCfgChildOf(parentCfg,"elecontent"),tryStr=this.#getCfgStrAttr(parentCfg,"try",!0,!1),tryTimes=tryStr?parseInt(tryStr):1,{elements:elements}=await this.#processElementCfgs(eleCfgs,tryTimes);if(selectedElements=elements,0===selectedElements.length){if("length"===this.#getCfgStrAttr(elecontentCfg,"type"))content="0";else if("element"===parentType&&parentCfg[appConfig.XML_ATTR_KEY].colname){const errName=this.#getCfgStrAttr(parentCfg,"errname");this.#procErrname(errName,"columnElement","invalidColname")}}else content=await this.#processElecontentCfg(elecontentCfg,elements)}else if("templstr"===contentSrcType){const templstrCfg=this.#getRequiredCfgChildOf(parentCfg,"templstr");content=this.#getCfgStrAttr(templstrCfg,"templ")}else if("property"===contentSrcType){const subKeys=this.#getCfgStrAttr(parentCfg,"subkeys"),keys=this.#getSubKeys(subKeys),obj=this.#getMemberOfObject(objItem,keys);content=this.#getStringOfObj(obj)}else if("get"===contentSrcType){const origGetCfg=this.#getRequiredCfgChildOf(parentCfg,"get"),getCfg=this.#transformStrBooleanAttrsInCfg(origGetCfg);let page=this.#taskContext.page,targetType="page";const otherInfo={paras:this.#paras},type=this.#getCfgStrAttr(origGetCfg,"type");if("screenshot"===type){const eleCfg=this.#getOptionalCfgChildOf(origGetCfg,"element");if(eleCfg){const{elements:elements}=await this.#processElementCfgs([eleCfg],1);if(!(elements.length>0))throw new Error("##te no elements selected in get_screenshot");targetType="element",content=await GetPara.getParaInElement(getCfg,elements[0],otherInfo)}}else if("httpheaders"===type){const headerMethod=getCfg.headermethod;otherInfo.interceptionData=this.#interceptionData,"genNewByHttpOptions"===headerMethod&&(otherInfo.genHeaderOptions=this.#getGenHttpHeaderOptions(getCfg,origGetCfg));const newHttpHeaders=[];this.#getCfgChildrenOf(origGetCfg,"httpheader",!0).forEach(headerCfg=>{let name=this.#getCfgStrAttr(headerCfg,"name");if(name=name.toLowerCase().trim(),name){const source=this.#getCfgStrAttr(headerCfg,"source"),value=this.#getCfgStrAttr(headerCfg,"value"),deleteFlag=this.#getCfgBoolAttr(headerCfg,"delete");newHttpHeaders.push({name:name,source:source,value:value,deleteFlag:deleteFlag})}}),otherInfo.newHttpHeaders=newHttpHeaders}else"response"!==type&&"special"!==type\|\|(otherInfo.interceptionData=this.#interceptionData);page&&"page"===targetType&&["content","cookies","mhtml","pdf","screenshot","title","window"].includes(getCfg.type)?content=await GetPara.getParaInPage(getCfg,page,otherInfo):["addedtaskparas","datetime","execedtaskparas","file","httpheaders","queuedcntwithparas","queuedtaskparas","random","response","special","sysparas"].includes(getCfg.type)?content=await GetPara.getPara(getCfg,otherInfo):["screenshot"].includes(getCfg.type)&&"element"===targetType\|\|this.#procErrname("cfginvalid","getPara","invalidType")}else if("subtask"===contentSrcType){const subtaskCfg=this.#getRequiredCfgChildOf(parentCfg,"subtask");loginfo(`##te to execsync subtaskCfg: ${JSON.stringify(subtaskCfg)}`),content=await this.#processSyncSubtask(subtaskCfg)}else{if("ocr"!==contentSrcType)throw new Error(`##te Invalid content source type: ${contentSrcType}`);{let ocrCfg=this.#getRequiredCfgChildOf(parentCfg,"ocrcfg");const eleCfgs=this.#getCfgChildrenOf(parentCfg,"element",!0),{elementSource:elementSource,elements:elements}=await this.#processElementCfgs(eleCfgs,1,!1);if(elements.length>0&&(this.#checkBrowserElementSource(elementSource,"ocr"),content=await this.__processBrowserOcrCfg(ocrCfg,elements),"number"==typeof content&&content<0))throw new Error(`##te Error with ocr content: ${content}`)}}const decodefontsvgCfg=this.#getOptionalCfgChildOf(parentCfg,"decodefontsvg"),decodefontttfCfg=this.#getOptionalCfgChildOf(parentCfg,"decodefontttf");(decodefontsvgCfg\|\|decodefontttfCfg)&&(content=await this.__decodeFontInContent(content,decodefontsvgCfg,decodefontttfCfg));const transformCfg=this.#getOptionalCfgChildOf(parentCfg,"transform");return transformCfg&&(content=await this.#processTransformCfg(transformCfg,content),null===content&&this.#procErrname("cfginvalid","decodeFont","invalidFontchars")),{content:content,elements:selectedElements}}catch(err){throw loginfo(`##te getEleOrTemplTransform: tId${this.#templateId} pCfg: ${JSON.stringify(parentCfg).slice(0,200)}`),loginfo(`##te err: ${JSON.stringify(err)}`),new Error(err)}}#getColumnStrValOfLastRec(tabName,columnName){let lastStrVal="";if(this.#taskContext.execData[tabName]&&this.#taskContext.execData[tabName].length>0){let recCnt=this.#taskContext.execData[tabName].length,lastVal=this.#taskContext.execData[tabName][recCnt-1][columnName];lastStrVal=String("boolean"==typeof lastVal?Number(lastVal):lastVal)}return lastStrVal}async#processColumnCfg(columnCfg,tableName,objItem){const valErrName=this.#getCfgStrAttr(columnCfg,"valerrname"),pattern=this.#getCfgStrAttr(columnCfg,"pattern");try{this.#validateCfgType(columnCfg);const columnType=this.#getCfgStrAttr(columnCfg,"type"),colName=this.#getCfgStrAttr(columnCfg,"colname");let{content:content}=await this.#getEleOrTemplTransform(columnType,columnCfg,objItem);if(0===content.length){this.#getCfgBoolAttr(columnCfg,"completed")&&(content=this.#getColumnStrValOfLastRec(tableName,colName))}if("ignore"!==valErrName&&pattern){const flags=this.#getCfgStrAttr(columnCfg,"flags"),re=this.#getRegExp(pattern,flags);re&&!content.match(re)&&this.#procErrname(valErrName,"columnContent","validate")}this.#getCfgBoolAttr(columnCfg,"setvar")&&this.#setNormalVariable(colName,content);const discardedFlag=this.#getCfgBoolAttr(columnCfg,"discarded"),retObj={};if(!discardedFlag){let retVal;const dataType=this.#getCfgStrAttr(columnCfg,"datatype");"number"===dataType?retVal=this.#getIntVal(content,"round"):"boolen"===dataType&&(retVal=this.#getBoolVal(content)),retObj[colName]="string"===dataType?content:retVal}return retObj}catch(err){throw this.#taskContext.credits>=0&&(loginfo(`##te processColumnCfg: tId${this.#templateId}, fCfg ${JSON.stringify(columnCfg).slice(0,100)}`),loginfo(`##te err: ${JSON.stringify(err)}`)),new Error(err)}}async#processExtractAction(actCfg){const tabName=this.#getCfgStrAttr(actCfg,"tabname");if("string"!=typeof tabName\|\|0===tabName.length)throw new Error(`##te Invalid table name: ${tabName}`);this.#taskContext.execData[tabName]\|\|(this.#taskContext.execData[tabName]=[]);const rec={},columnCfgs=this.#getCfgChildrenOf(actCfg,"column",!0);for(let columnCfg of columnCfgs){const colName=this.#getCfgStrAttr(columnCfg,"colname");if(!(this.#extractColumns.length>0)\|\|this.#extractColumns.includes(colName))try{let resObj=await this.#processColumnCfg(columnCfg,tabName);Object.assign(rec,resObj),colName.startsWith("cx")&&loginfo(`##dbg extract: ${colName}=${rec[colName]}`)}catch(err){throw this.#taskContext.credits>=0&&loginfo(`##te extract column tId${this.#templateId} fCfg: ${JSON.stringify(columnCfg).slice(0,200)}`),err}}return this.#taskContext.execData[tabName].push(rec),!0}#getMemberOfObject(obj,keys){let result=obj;if(0===keys.length)return result;for(const key of keys)if(result=result?.[key],void 0===result)return;return result}#getStringOfObj(obj){try{switch(typeof obj){case"string":return obj;case"number":case"bigint":return String(obj);case"boolean":return obj?"1":"0";case"object":return obj?JSON.stringify(obj):"";default:return""}}catch(err){return loginfo(err),""}}#checkIsNotNullObject(obj,errName){obj&&"object"==typeof obj\|\|this.#procErrname(errName,"actionExtractArray","notNormalObject")}#getSubKeys(subKeysStr){if(!subKeysStr.trim())return[];const keys=subKeysStr.trim().split(".");if(!keys.every(key=>!!key))throw this.#procErrname("cfginvalid","actionExtractArray","invalidSubkeys"),new Error("unreachable");return keys}#getArrayItemsFromCachedResponses(reqUrl,subKeys,errName){const arrayItems=[],keys=this.#getSubKeys(subKeys);let responses=this.#interceptionData.responses;if(reqUrl){const re=new RegExp(reqUrl);responses=responses.filter(r=>r.requestUrl.match(re))}for(const response of responses){const content=response.responseData,json=(0,import_utils9.getJsonParseOfStr)(content,null);this.#checkIsNotNullObject(json,errName);const objOrItems=this.#getMemberOfObject(json,keys);if(this.#checkIsNotNullObject(objOrItems,errName),Array.isArray(objOrItems)){if(!objOrItems.every(item=>item&&"object"==typeof item)){this.#procErrname(errName,"actionExtractArray","invalidResponseItems");continue}objOrItems.forEach(item=>arrayItems.push(item))}else objOrItems&&"object"==typeof objOrItems&&arrayItems.push(objOrItems)}return arrayItems}#getArrayItemsFromString(list,subKeys,errName){let arrayItems=(0,import_utils9.getJsonParseOfStr)(list);if(this.#checkIsNotNullObject(arrayItems,errName),subKeys){const keys=this.#getSubKeys(subKeys);arrayItems=this.#getMemberOfObject(arrayItems,keys),this.#checkIsNotNullObject(arrayItems,errName)}return Array.isArray(arrayItems)?arrayItems.every(item=>item&&"object"==typeof item)?arrayItems:(this.#procErrname(errName,"actionExtractArray","invalidArrayItems"),[]):arrayItems&&"object"==typeof arrayItems?[arrayItems]:[]}async#processExtractArrayAction(actCfg){const{list:list,requrl:requrl,subkeys:subkeys,keys:keys,tabname:tabName,varname:varName,idxname:idxName,maxloops:maxloops,errname:errName}=this.#transformStrBooleanAttrsInCfg(actCfg);let arrayItems=list?this.#getArrayItemsFromString(list,subkeys,errName):this.#getArrayItemsFromCachedResponses(requrl,subkeys,errName);const maxLoops=parseInt(maxloops);maxLoops>0&&arrayItems.length>maxLoops&&(arrayItems=arrayItems.slice(0,maxLoops));let records=this.#taskContext.execData[tabName];if(records\|\|(records=[],this.#taskContext.execData[tabName]=records),0===arrayItems.length)return!0;const columnCfgs=this.#getCfgChildrenOf(actCfg,"column",!1),objKeys=keys?keys.split(","):0===columnCfgs.length?Array.from(Object.keys(arrayItems[0])):[];for(const[idx,objItem]of arrayItems.entries()){const rec={};varName&&this.#setNormalVariable(varName,JSON.stringify(objItem)),idxName&&this.#setNormalVariable(idxName,String(idx));for(const key of objKeys){const obj=this.#getMemberOfObject(objItem,[key]),colValue=this.#getStringOfObj(obj);rec[key]=colValue}for(let columnCfg of columnCfgs)try{const colName=this.#getCfgStrAttr(columnCfg,"colname");let resObj=await this.#processColumnCfg(columnCfg,tabName,objItem);Object.assign(rec,resObj),colName.startsWith("cx")&&loginfo(`##dbg extract: ${colName}=${rec[colName]}`)}catch(err){throw this.#taskContext.credits>=0&&loginfo(`##te extract column tId${this.#templateId} fCfg: ${JSON.stringify(columnCfg).slice(0,200)}`),err}records.push(rec)}return!0}async#getExtractScriptInData(maxLoops,htmlFlag,tabName,errName){try{const vars=Object.assign({},this.#paras,{authInfo:null,sysParas:null}),execData=this.#taskContext.execData,data={vars:vars,responses:this.#interceptionData.responses,tabName:tabName,maxLoops:maxLoops,errName:errName,execData:execData};if(htmlFlag&&this.#taskContext.page){const html=await this.#taskContext.page.content();data.html=html}return data}catch(err){return null}}#checkExtractScriptResult(result){if(!result\|\|"object"!=typeof result\|\|!result.execData\|\|"object"!=typeof result.execData)return!1;const{execData:execData,errName:errName}=result;for(const tabName of Object.keys(execData))if(!tabName.startsWith("dat_")\|\|20!==tabName.length)return!1;for(const records of Object.values(execData))if(!Array.isArray(records)\|\|!records.every(rec=>rec&&"object"==typeof rec))return!1;return!!(void 0===errName\|\|"string"==typeof errName&&errName)}async#processExtractScriptAction(actCfg){const cfg=this.#transformStrBooleanAttrsInCfg(actCfg),{base64:base64Flag,html:htmlFlag,isolated:isolatedFlag,tabname:tabName,maxloops:maxLoopsStr,errname:errName}=cfg;let script=this.#getCfgTextContext(actCfg).trim();if(!script)return!0;if(base64Flag&&(script=(0,import_utils9.decodeFromBase64)(script)),script.length<10\|\|!isolatedFlag)return this.#procErrname("cfginvalid","actionExtractScript","invalidScript"),!1;try{const maxLoops=parseInt(maxLoopsStr),inData=await this.#getExtractScriptInData(maxLoops,htmlFlag,tabName,errName);if(!inData)return this.#procErrname(errName,"actionExtractScript","invalidInData"),!1;const isolate=new import_isolated_vm2.default.Isolate,context=isolate.createContextSync(),externalCopy=new import_isolated_vm2.default.ExternalCopy(inData);context.global.setSync("inData",externalCopy.copyInto());const content=isolate.compileScriptSync(script).runSync(context);"string"==typeof content&&content\|\|logwarn("Please check the script of extract is correct.");const result=JSON.parse(content);if(!this.#checkExtractScriptResult(result))return this.#procErrname(errName,"actionExtractScript","invalidResult"),!1;result.errName&&("ignore"===result.errName?this.#procErrname(errName,"actionExtractScript","resultError1"):this.#procErrname(result.errName,"actionExtractScript","resultError2"));const execData=this.#taskContext.execData,newExecData=result.execData;for(const tabName2 of Object.keys(newExecData))execData[tabName2]=newExecData[tabName2];return!0}catch(err){return logwarn(err.message),this.#procErrname(errName,"actionExtractScript","pesaException"),!1}}async#isTableExtractable(vertical){let attrName="rowspan",eleCfg=this.#generateElementCfg(`[${attrName}]`);const{elements:rowElements}=await this.#processElementCfg(eleCfg);if(rowElements.length>0)for(const element of rowElements){const rowspan=await element.attribute(attrName);if(!rowspan&&parseInt(rowspan)>1)return!1}attrName="colspan",eleCfg=this.#generateElementCfg(`[${attrName}]`);const{elements:colElements}=await this.#processElementCfg(eleCfg);if(colElements.length>0)for(const element of colElements){const colspan=await element.attribute(attrName);if(!colspan&&parseInt(colspan)>1)return!1}return!0}#getTableHeader(origHeader,keyformat,colname,onlyDigitalCharFlag=!0){let headerStr=onlyDigitalCharFlag&&"camel"===keyformat?origHeader.replaceAll(/[^\w\|\s]/g,"").replaceAll(/_/g," "):origHeader;if(colname&&"string"==typeof colname)return colname;if("header"===keyformat)return headerStr;{const words=headerStr.split(/\s+/);let header=words[0].toLowerCase();for(const word of words.slice(1))header=`${header}${word[0].toUpperCase()}${word.slice(1).toLowerCase()}`;return header}}async#extractHorizontalTable(keyformat,colnames,tabname,maxLoops,errName){const headers=[],records=[];let hasThead=!1,spaceIdx=1,trLocPrefix="",recordStartIdx=1,eleCfg=this.#generateElementCfg("./thead/tr");const{elements:theadTrElements}=await this.#processElementCfg(eleCfg);if(theadTrElements.length>1)return this.#procErrname(errName,"actionExtractTable","manyTheadTrs"),!0;if(1===theadTrElements.length){hasThead=!0,eleCfg=this.#generateElementCfg("./thead/tr/th");const{elements:theadTrThElements}=await this.#processElementCfg(eleCfg);if(0===theadTrThElements.length)return this.#procErrname(errName,"actionExtractTable","noTheadTrTh"),!0;for(const[idx,th]of theadTrThElements.entries()){const origHeader=(await th.textContent()).trim(),header=this.#getTableHeader(origHeader\|\|"space"+spaceIdx++,keyformat,colnames[idx]);headers.push(header)}}trLocPrefix="./tr",eleCfg=this.#generateElementCfg(trLocPrefix);let{elements:trElements}=await this.#processElementCfg(eleCfg);if(0===trElements.length){trLocPrefix="./tbody/tr",eleCfg=this.#generateElementCfg(trLocPrefix);let{elements:tbodyTrElements}=await this.#processElementCfg(eleCfg);trElements=tbodyTrElements}if(trElements.length<=1)return this.#procErrname(errName,"actionExtractTable","noRecord"),!0;if(!hasThead){eleCfg=this.#generateElementCfg(`${trLocPrefix}[${recordStartIdx}]/`);let{elements:thElements}=await this.#processElementCfg(eleCfg);if(0===thElements.length)return this.#procErrname(errName,"actionExtractTable","noTr"),!0;for(const[idx,th]of thElements.entries()){const origHeader=(await th.textContent()).trim(),header=this.#getTableHeader(origHeader\|\|"space"+spaceIdx++,keyformat,colnames[idx]);headers.push(header)}recordStartIdx++,trElements=trElements.slice(1)}const headersCnt=headers.length;for(const recordIdx of trElements.keys()){eleCfg=this.#generateElementCfg(`${trLocPrefix}[${recordStartIdx+recordIdx}]/`);let{elements:colElements}=await this.#processElementCfg(eleCfg);if(colElements.length!==headersCnt)return this.#procErrname(errName,"actionExtractTable","differentColumns"),!0;const record={};for(const colIdx of colElements.keys()){if(maxLoops>0&&colIdx>maxLoops)break;const val=await colElements[colIdx].textContent();record[headers[colIdx]]=val}records.push(record)}const origData=this.#taskContext.execData[tabname];return Array.isArray(origData)?records.forEach(record=>origData.push(record)):this.#taskContext.execData[tabname]=records,!0}async#extractVerticalTable(keyformat,colnames,tabname,maxLoops,errName){const records=[];let recordsCnt=0,trLocPrefix="",spaceIdx=1;trLocPrefix="./tr";let eleCfg=this.#generateElementCfg(trLocPrefix),{elements:trElements}=await this.#processElementCfg(eleCfg);if(0===trElements.length){trLocPrefix="./tbody/tr",eleCfg=this.#generateElementCfg(trLocPrefix);let{elements:tbodyTrElements}=await this.#processElementCfg(eleCfg);trElements=tbodyTrElements}if(0===trElements.length)return this.#procErrname(errName,"actionExtractTable","noTrElments"),!0;for(const trIdx of trElements.keys()){let thCfg=this.#generateElementCfg(`${trLocPrefix}[${trIdx+1}]/th`),{elements:thElements}=await this.#processElementCfg(thCfg);if(1!==thElements.length)return this.#procErrname(errName,"actionExtractTable","noThElements"),!0;const origHeader=(await thElements[0].textContent()).trim(),header=this.#getTableHeader(origHeader\|\|"space"+spaceIdx++,keyformat,colnames[trIdx]);let tdCfg=this.#generateElementCfg(`${trLocPrefix}[${trIdx+1}]/td`),{elements:tdElements}=await this.#processElementCfg(tdCfg);if(0===tdElements.length)return this.#procErrname(errName,"actionExtractTable","noTdElements"),!0;if(0===trIdx){for(const tdIdx of tdElements.keys()){if(maxLoops>0&&tdIdx>maxLoops)break;const val=await tdElements[tdIdx].textContent(),record={};record[header]=val,records.push(record)}recordsCnt=records.length}else{if(tdElements.length<recordsCnt)return this.#procErrname(errName,"actionExtractTable","differentVertCols"),!0;for(const tdIdx of tdElements.keys()){if(tdIdx>=recordsCnt)break;const val=await tdElements[tdIdx].textContent();records[tdIdx][header]=val}}}const origData=this.#taskContext.execData[tabname];return Array.isArray(origData)?records.forEach(record=>origData.push(record)):this.#taskContext.execData[tabname]=records,!0}async#processExtractTableAction(actCfg){const{orientation:orientation,check:check,tabname:tabname,keyformat:keyformat,colnames:colnamesStr,maxloops:maxloops,errname:errName}=this.#transformStrBooleanAttrsInCfg(actCfg),colnames=colnamesStr?colnamesStr.split(","):[],eleCfgs=this.#getCfgChildrenOf(actCfg,"element",!0),maxLoops=parseInt(maxloops);if(maxLoops<0)return!1;if(0===eleCfgs.length)return this.#procErrname("logicerror","actionExtractTable","noEleCfg"),!1;const{elementSource:elementSource,elements:tables}=await this.#processElementCfg(eleCfgs[0]);if(0===tables.length)return this.#procErrname(errName,"actionExtractTable","noTables"),!0;const element=tables[0];if("browser"===elementSource?this.#taskContext.browserElesStack.push(element):"cheerio"===elementSource?this.#taskContext.cheerioElesStack.push(element):(0,import_utils9.unreachable)(elementSource),this.#taskContext.loopsStack.push(element),check){if(!await this.#isTableExtractable("vertical"===orientation))return this.#procErrname("cfginvalid","actionExtractTable","notExtractable"),!1}return"vertical"===orientation?await this.#extractVerticalTable(keyformat,colnames,tabname,maxLoops,errName):await this.#extractHorizontalTable(keyformat,colnames,tabname,maxLoops,errName),this.#taskContext.loopsStack.pop(),"browser"===elementSource?this.#taskContext.browserElesStack.pop():"cheerio"===elementSource?this.#taskContext.cheerioElesStack.pop():(0,import_utils9.unreachable)(elementSource),!0}async#performExtractAction(actCfg){this.#validateCfgType(actCfg);let subType=this.#getCfgStrAttr(actCfg,"subtype",!1,!1);if(subType)if("array"===subType)await this.#processExtractArrayAction(actCfg);else if("script"===subType)await this.#processExtractScriptAction(actCfg);else{if("table"!==subType)return this.#procErrname("cfginvalid","actionExtract","logicError"),!1;await this.#processExtractTableAction(actCfg)}else await this.#processExtractAction(actCfg);return!0}#getHttpHeadersObj(headers){try{const headersObj=JSON.parse(headers);return headersObj&&"object"==typeof headersObj?headersObj:null}catch(err){this.#procErrname("cfginvalid","actionGoto","invalidHeaders")}}async#performGotoAction(actCfg){const page=this.#taskContext.page;if(!page)return this.#procErrname("cfginvalid","actionGogo","noPage"),!1;this.#taskContext.browserElesStack.length>0&&this.#procErrname("cfginvalid","actionGoto","inLoopOfBrowserElements");let url="";try{this.#validateCfgType(actCfg);const actType=this.#getCfgStrAttr(actCfg,"type");url=this.#getCfgStrAttr(actCfg,"url");const waitMsStr=this.#getCfgStrAttr(actCfg,"wait"),reuseFlag=this.#getCfgBoolAttr(actCfg,"reuse"),encodeFlag=this.#getCfgBoolAttr(actCfg,"encodeuri"),headers=this.#getCfgStrAttr(actCfg,"headers"),referer=this.#getCfgStrAttr(actCfg,"referer");if(encodeFlag&&(url=encodeURI(url)),!this.#validateUrl(url))throw new Error(`##te Invalid url: ${url} when trying to open this page`);if(loginfo(`##te goto url ${url}`),reuseFlag){if(page.url().includes(url))return await this.#processWait(waitMsStr),this.#addTaskCredits(actCfg),page}if(this.#lastUrl=url,headers.trim()){const httpHeadersObj=this.#getHttpHeadersObj(headers.trim());httpHeadersObj&&await page.setExtraHTTPHeaders(httpHeadersObj)}this.#canResetPage(page,actType);const dataLoginCaptchaInfo=this.#getDataLoginCaptchaInfo(actCfg,actType);return dataLoginCaptchaInfo&&dataLoginCaptchaInfo.popupFlag?await this.processNewPageLoginCaptcha(dataLoginCaptchaInfo,async(p,u,r)=>{r?await p.goto(u,{waitUntil:"load",timeout:0,referer:r}):await p.goto(u,{waitUntil:"load",timeout:0})},[page,url,referer]):await page.goto(url),dataLoginCaptchaInfo&&dataLoginCaptchaInfo.currentFlag&&await this.#processPageLoginCaptcha(page,dataLoginCaptchaInfo,"current"),await this.#processWait(waitMsStr),this.#addTaskCredits(actCfg),page}catch(err){if(loginfo(`##te kId${this.#taskId} goto: ${url}`),-46!==this.#taskContext.credits&&(logerr(`##te kId${this.#taskId} error credits ${this.#taskContext.credits} in performGotoAction`),logerr(err)),this.#taskContext.credits>=0){const message=err.message?err.message:"";if(message.includes("net::ERR_INTERNET_DISCONNECTED"))this.#setFailedCredits(-71);else if(message.includes("net::ERR_NAME_NOT_RESOLVED")){await(0,import_utils9.checkDns)(appConfig.CHECK_DNS_DOMAIN)?this.#setFailedCredits(-69):this.#setFailedCredits(-71)}else message.includes("net::ERR_CONNECTION_TIMED_OUT")?this.#setFailedCredits(-72):message.includes("net::ERR_CONNECTION_REFUSED")\|\|message.includes("net::ERR_CONNECTION_RESET")?this.#setFailedCredits(-98):message.includes("net::ERR_PROXY_CONNECTION_FAILED ")\|\|message.includes("net::ERR_EMPTY_RESPONSE")\|\|message.includes("net::ERR_TIMED_OUT")\|\|message.includes("net::ERR_TUNNEL_CONNECTION_FAILED")?this.#setFailedCredits(-86):message.includes("net::ERR_INVALID_AUTH_CREDENTIALS")?this.#setFailedCredits(-88):(message.includes("Navigation timeout"),this.#setFailedCredits(-85))}throw new Error(err)}}async#handleHover(element,tryTimes=10){let error=null;for(let i=0;i<tryTimes;i++)try{return await element.hover(),!0}catch(err){loginfo(err),await(0,import_utils9.sleep)(2e3),error=err}throw error}async#performHoverAction(actCfg){const page=this.#taskContext.page;if(!page)return this.#procErrname("cfginvalid","actionHover","noPage"),!1;this.#validateCfgType(actCfg);const tryTimes=parseInt(this.#getCfgStrAttr(actCfg,"try")),errName=this.#getCfgStrAttr(actCfg,"errname");try{const eleCfgs=this.#getCfgChildrenOf(actCfg,"element",!0),{elementSource:elementSource,elements:elements}=await this.#processElementCfgs(eleCfgs,tryTimes);if(this.#checkBrowserElementSource(elementSource,"hover"),0===elements.length)return this.#procErrname(errName,"actionHover","noElements"),page;this.#isHeadless()&&await page.bringToFront(),logdbg(`##te pageIdx${this.#getPageInfo().pageIdx} hover`),await this.#handleHover(elements[0]);const waitMsStr=this.#getCfgStrAttr(actCfg,"wait");return await this.#processWait(waitMsStr),this.#addTaskCredits(actCfg),page}catch(err){return loginfo(err),this.#procErrname("other","actionHover","phaException"),page}}async#processConditionCfg(conditionCfg){this.#validateCfgType(conditionCfg);const conditionType=this.#getCfgStrAttr(conditionCfg,"type");let conditionResult=!1;if(["element","templstr"].includes(conditionType)){let content="";content=(await this.#getEleOrTemplTransform(conditionType,conditionCfg)).content,conditionResult=this.#getBoolVal(content)}else{if("else"!==conditionType)throw new Error("##te Invalid type(condition)");conditionResult=!0}if(conditionResult){const subactions=this.#getCfgChildrenOf(conditionCfg,"action");await this.#performActions(subactions)}return conditionResult}async#performIfelseAction(actCfg){this.#validateCfgType(actCfg);const conditionCfgs=this.#getCfgChildrenOf(actCfg,"condition");for(let conditionCfg of conditionCfgs)if(await this.#processConditionCfg(conditionCfg))break;return!0}async#performInputAction(actCfg){if(!this.#taskContext.page)return this.#procErrname("cfginvalid","actionInput","noPage"),!1;this.#validateCfgType(actCfg);const content=this.#getCfgStrAttr(actCfg,"content"),replace=this.#getCfgBoolAttr(actCfg,"replace"),enter=this.#getCfgBoolAttr(actCfg,"enter"),tryTimes=parseInt(this.#getCfgStrAttr(actCfg,"try")),errName=this.#getCfgStrAttr(actCfg,"errname"),eleCfgs=this.#getCfgChildrenOf(actCfg,"element",!0);try{const{elementSource:elementSource,elements:elements}=await this.#processElementCfgs(eleCfgs,tryTimes);if(this.#checkBrowserElementSource(elementSource,"input"),0===elements.length)return this.#procErrname(errName,"actionInput","noElements"),!1;await elements[0].input(content,{replace:replace,enter:enter});const waitMsStr=this.#getCfgStrAttr(actCfg,"wait");return await this.#processWait(waitMsStr),this.#addTaskCredits(actCfg),!0}catch(err){return logerr(err),!1}}#transformStrBooleanAttrsInCfg(origCfg,varFlag=!0){const obj=Object.assign({},origCfg[appConfig.XML_ATTR_KEY]);if(!obj\|\|"object"!=typeof obj)throw new Error("Invalid origCfg in transformStrAttrsInCfg");if(!varFlag)return obj;for(const key of Object.keys(obj))"string"==typeof obj[key]&&obj[key].includes("${")&&(obj[key]=this.#transTemplToStr(obj[key],this.#paras));return obj}#getRequestInterceptionOptions(requestCfgs){const options=[];for(const requestCfg of requestCfgs){const{type:type,method:method,url:url,resourcetype:resourcetype,postdata:postdata}=requestCfg,requestMatch={};if(method&&"all"!==method&&(requestMatch.methods=[method.toUpperCase()]),url&&(requestMatch.url=new RegExp(url)),resourcetype&&"all"!==resourcetype&&(requestMatch.resourceTypes=[resourcetype]),postdata&&(requestMatch.postData=new RegExp(postdata)),"abort"===type){const option={requestMatch:requestMatch,action:"abort"};options.push(option)}else{if("fufill"!==type)throw new Error(`Invalid action type ${type}`);{const{fulfill:fulfill}=requestCfg,option={requestMatch:requestMatch,action:"fulfill",fulfill:fulfill};options.push(option)}}}return options}#getResponseInterceptionOptions(responseCfgs){const options=[];for(const responseCfg of responseCfgs){const{type:type,method:method,url:url,resourcetype:resourcetype,postdata:postdata,minsize:minsize,maxsize:maxsize}=responseCfg,requestMatch={};method&&"all"!==method&&(requestMatch.methods=[method.toUpperCase()]),url&&(requestMatch.url=new RegExp(url)),resourcetype&&"all"!==resourcetype&&(requestMatch.resourceTypes=[resourcetype]),postdata&&(requestMatch.postData=new RegExp(postdata));const responseMatch={},minSize=parseInt(minsize);minSize>0&&(responseMatch.minLength=minSize);const maxSize=parseInt(maxsize);if(maxSize>0&&(responseMatch.maxLength=maxSize),"cache"===type){const{tabname:tabname,contenttype:contenttype,cache:cacheFlag,requestheaders:requestHeadersFlag,responseheaders:responseHeadersFlag}=responseCfg,option={requestMatch:requestMatch,responseMatch:responseMatch,handler:responseCacheCb,handlerOptions:{execData:this.#taskContext.execData,contentType:contenttype,tabname:tabname,cacheFlag:cacheFlag,requestHeadersFlag:requestHeadersFlag,responseHeadersFlag:responseHeadersFlag,interceptionData:this.#interceptionData}};options.push(option)}else{if("save"!==type)throw new Error(`Invalid response type ${type}`);{const{basedir:basedir,encoding:encoding,extname:extname,hashmethod:hashmethod,hostnameparts:hostnameparts,params:params,pathtype:pathtype,tabname:tabname,validextnames:validextnames}=responseCfg,option={requestMatch:requestMatch,responseMatch:responseMatch,handler:responseSaveCb,handlerOptions:{execData:this.#taskContext.execData,baseDir:basedir,pathType:pathtype,hashMethod:hashmethod,hostnameParts:parseInt(hostnameparts),extname:extname,validExtnames:validextnames,params:params,encoding:encoding,tabname:tabname}};options.push(option)}}}return options}async#performInterceptAction(actCfg){const page=this.#taskContext.page;if(!page)return this.#procErrname("cfginvalid","actionIntercept","noPage"),!1;this.#validateCfgType(actCfg);const subType=this.#getCfgStrAttr(actCfg,"subtype");let requestCfgs=this.#getCfgChildrenOf(actCfg,"request"),responseCfgs=this.#getCfgChildrenOf(actCfg,"response");try{if(requestCfgs=requestCfgs.map(requestCfg=>this.#transformStrBooleanAttrsInCfg(requestCfg)),responseCfgs=responseCfgs.map(responseCfg=>this.#transformStrBooleanAttrsInCfg(responseCfg)),"set"===subType){if(requestCfgs.length<=0&&responseCfgs.length<=0&&this.#procErrname("cfginvalid","actionIntercept","noCfgs"),requestCfgs.length>0){const options=this.#getRequestInterceptionOptions(requestCfgs);await page.setRequestInterception(options)}if(responseCfgs.length>0){const options=this.#getResponseInterceptionOptions(responseCfgs);await page.setResponseInterception(options)}}else"clear"===subType&&(await page.clearRequestInterceptions(),await page.clearResponseInterceptions());return!0}catch(err){return logerr(err),!1}}async#performLoopforAction(actCfg){this.#validateCfgType(actCfg);try{const errName=this.#getCfgStrAttr(actCfg,"errname");let roundType=this.#getCfgStrAttr(actCfg,"roundtype"),fromInt=this.#getIntVal(this.#getCfgStrAttr(actCfg,"from"),roundType),toInt=this.#getIntVal(this.#getCfgStrAttr(actCfg,"to"),roundType),stepInt=this.#getIntVal(this.#getCfgStrAttr(actCfg,"step"),roundType),varName=this.#getCfgStrAttr(actCfg,"varname"),maxLoops=this.#getIntVal(this.#getCfgStrAttr(actCfg,"maxloops"),roundType);if(maxLoops<0)return!1;let loops=0;for(let i=fromInt;i<=toInt&&!(maxLoops&&loops++>=maxLoops);i+=stepInt){this.#taskContext.loopsStack.push(i),this.#setNormalVariable(varName,String(i));const subactions=this.#getCfgChildrenOf(actCfg,"action");if(await this.#performActions(subactions),this.#taskContext.loopsStack.pop(),this.#taskContext.continueFlag&&(this.#taskContext.continueFlag=!1),this.#taskContext.breakFlag){this.#taskContext.breakFlag=!1;break}}return loops<1&&this.#procErrname(errName,"actionLoopfor","zeroLoops"),!0}catch(err){throw err}}async#processElementCfg(eleCfg){try{const selector=this.#getCfgStrAttr(eleCfg,"loc").trim(),elementSource=this.#getElementSource(eleCfg),absoluteFlag=this.#getCfgBoolAttr(eleCfg,"absolute"),iframeOptions=this.#getCfgChildrenOf(eleCfg,"iframe",!1).map(cfg=>this.#transformStrBooleanAttrsInCfg(cfg)).map(cfg=>{const{srcprefix:src,loc:selector2}=cfg;return{src:src,selector:selector2}}),browserPage=this.#taskContext.page,retObj={elementSource:elementSource,elements:[]};if("browser"===elementSource){if(!browserPage)throw new Error("Cannot get elements from null browser page");if(0===this.#taskContext.browserElesStack.length)retObj.elements=await browserPage.findElements(selector,iframeOptions);else{const parent=this.#taskContext.browserElesStack.slice(-1)[0];retObj.elements=await parent.findElements(selector,iframeOptions,absoluteFlag)}return retObj}if("cheerio"===elementSource){if(iframeOptions.length>0)throw new Error("Iframe config is invalid in cheerio source");let cheerioPage=this.#taskContext.cheerioPage;if(!cheerioPage){if(!browserPage)throw new Error("Please set CheerioPage first, from which to select elememnts");const content=await browserPage.content();content.length>10&&(this.#setCheerioPage(content,!0),cheerioPage=this.#taskContext.cheerioPage)}if(!cheerioPage)throw new Error("Cannot get elements from null cheerioPage");if(0===this.#taskContext.cheerioElesStack.length)retObj.elements=await cheerioPage.findElements(selector);else{const parent=this.#taskContext.cheerioElesStack.slice(-1)[0];retObj.elements=await parent.findElements(selector,iframeOptions,absoluteFlag)}return retObj}(0,import_utils9.unreachable)(elementSource)}catch(err){throw loginfo(`##te processElementCfg: tId${this.#templateId} eleCfg: ${JSON.stringify(eleCfg).slice(0,200)}`),new Error(err)}}async#processElementCfgs(eleCfgs,tryTimes=1,firstSelectedFlag=!0){try{if(0===eleCfgs.length)throw new Error("No element tags");const retObj={elementSource:"browser",elements:[]};for(let i=0;i<tryTimes;i++){let lastElementSource="";const selectedElements=[];for(const eleCfg of eleCfgs){const{elementSource:elementSource,elements:elements}=await this.#processElementCfg(eleCfg);if(lastElementSource&&lastElementSource!=elementSource)throw new Error("Element sources of all element tags must be the same");if(lastElementSource=elementSource,elements.length>0&&(elements.forEach(e=>selectedElements.push(e)),firstSelectedFlag))break}if(lastElementSource&&(retObj.elementSource=lastElementSource),selectedElements.length>0)return retObj.elements=selectedElements,retObj;i>0&&(loginfo(`##tmp ${i} try failed in getEleOrTemplTransform`),await(0,import_utils9.sleep)(1e3))}return retObj}catch(err){throw loginfo(`##te processElementCfgs: eleCfgs: ${JSON.stringify(eleCfgs).slice(0,200)}`),new Error(err)}}async#performLoopinelesAction(actCfg){if(!this.#taskContext.page)return this.#procErrname("cfginvalid","actionLoopineles","noPage"),!1;this.#validateCfgType(actCfg);const varName=this.#getCfgStrAttr(actCfg,"varname"),positionFlag=varName.startsWith("position"),errName=this.#getCfgStrAttr(actCfg,"errname"),eleCfgs=this.#getCfgChildrenOf(actCfg,"element",!0);let maxLoops=this.#getIntVal(this.#getCfgStrAttr(actCfg,"maxloops"),"round"),start=this.#getIntVal(this.#getCfgStrAttr(actCfg,"start"),"round"),end=this.#getIntVal(this.#getCfgStrAttr(actCfg,"end"),"round"),step=this.#getIntVal(this.#getCfgStrAttr(actCfg,"step"),"round");if(step<1&&(step=1),maxLoops<0)return!1;const{elementSource:elementSource,elements:elements}=await this.#processElementCfgs(eleCfgs,1),selectedElesCnt=elements.length;if(0===selectedElesCnt)return this.#procErrname(errName,"actionLoopineles","noElements"),!0;start<=0&&(start+=selectedElesCnt),end<=0?end+=selectedElesCnt:end>selectedElesCnt&&(end=selectedElesCnt);let processedNum=0;for(let i=start-1;i<end&&!(maxLoops&&processedNum>=maxLoops);i+=step){varName&&this.#setNormalVariable(varName,String(positionFlag?i+1:i));const element=elements[i];"browser"===elementSource?this.#taskContext.browserElesStack.push(element):"cheerio"===elementSource?this.#taskContext.cheerioElesStack.push(element):(0,import_utils9.unreachable)(elementSource),this.#taskContext.loopsStack.push(element);const subactions=this.#getCfgChildrenOf(actCfg,"action");if(await this.#performActions(subactions),this.#taskContext.loopsStack.pop(),"browser"===elementSource?this.#taskContext.browserElesStack.pop():"cheerio"===elementSource?this.#taskContext.cheerioElesStack.pop():(0,import_utils9.unreachable)(elementSource),processedNum++,this.#taskContext.continueFlag&&(this.#taskContext.continueFlag=!1),this.#taskContext.breakFlag){this.#taskContext.breakFlag=!1;break}}return!0}async#performLoopinstrAction(actCfg){this.#validateCfgType(actCfg);let regStr=this.#getCfgStrAttr(actCfg,"split");const trimFlag=this.#getCfgBoolAttr(actCfg,"trim"),errName=this.#getCfgStrAttr(actCfg,"errname");let regExp;try{regExp=new RegExp(regStr)}catch(error){throw new Error(`##te Invalid regexp: ${regExp}`)}const loopItems=trimFlag?this.#getCfgStrAttr(actCfg,"list").trim().split(regExp).map(item=>item.trim()).filter(item=>item.length>0):this.#getCfgStrAttr(actCfg,"list").split(regExp),varName=this.#getCfgStrAttr(actCfg,"varname");let maxLoops=this.#getIntVal(this.#getCfgStrAttr(actCfg,"maxloops"),"round");if(maxLoops<0)return!1;let loops=0;for(const item of loopItems){if(maxLoops&&loops++>=maxLoops)break;this.#taskContext.loopsStack.push(item),this.#setNormalVariable(varName,item);const subactions=this.#getCfgChildrenOf(actCfg,"action");if(await this.#performActions(subactions),this.#taskContext.loopsStack.pop(),this.#taskContext.continueFlag&&(this.#taskContext.continueFlag=!1),this.#taskContext.breakFlag){this.#taskContext.breakFlag=!1;break}}return loops<1&&this.#procErrname(errName,"actionLoopinstr","zeroLoops"),!0}async#performLoopdowhileAction(actCfg){this.#validateCfgType(actCfg);const subType=this.#getCfgStrAttr(actCfg,"subtype"),isWhileFlag=this.#getCfgBoolAttr(actCfg,"iswhile"),varName=this.#getCfgStrAttr(actCfg,"varname"),clickFlag="element"===subType&&this.#getCfgBoolAttr(actCfg,"click"),genFlag=(this.#getCfgBoolAttr(actCfg,"navigate"),"element"===subType&&this.#getCfgBoolAttr(actCfg,"gen")),capFlag="element"===subType&&this.#getCfgBoolAttr(actCfg,"cap"),errName=clickFlag?this.#getCfgStrAttr(actCfg,"errname"):"ignore",waitMsStr=clickFlag?this.#getCfgStrAttr(actCfg,"wait"):"-1";let maxLoops=this.#getIntVal(this.#getCfgStrAttr(actCfg,"maxloops"),"round");if(maxLoops<0)return!1;let{content:content,elements:elements}=await this.#getEleOrTemplTransform(subType,actCfg),loops=0;for(;0===loops&&!isWhileFlag\|\|this.#getBoolVal(content);){varName&&this.#setNormalVariable(varName,String(loops)),"element"!==subType\|\|!clickFlag\|\|this.#skipActions.includes("click")\|\|this.#skipActions.includes("operate")\|\|(elements.length<=0?this.#procErrname(errName,"actionLoopdowhile","noElements"):(await elements[0].click(),this.#addTaskCredits(actCfg,genFlag,capFlag))),this.#taskContext.loopsStack.push(loops);const subactions=this.#getCfgChildrenOf(actCfg,"action");if(await this.#performActions(subactions),this.#taskContext.loopsStack.pop(),this.#taskContext.continueFlag&&(this.#taskContext.continueFlag=!1),this.#taskContext.breakFlag){this.#taskContext.breakFlag=!1;break}if(await this.#processWait(waitMsStr),loops++,maxLoops&&loops>=maxLoops)break;let retObj=await this.#getEleOrTemplTransform(subType,actCfg);content=retObj.content,elements=retObj.elements}return!0}async#performGetstatedataInMisc(miscCfg){const cookiesSource=this.#getCfgStrAttr(miscCfg,"cookies"),localStorageSource=this.#getCfgStrAttr(miscCfg,"localstorage"),headersSource=this.#getCfgStrAttr(miscCfg,"headers"),userDataSource=this.#getCfgStrAttr(miscCfg,"userdata"),{cookies:cookies,localStorage:localStorage,userData:userData}=this.#taskContext.origStateData,stateData={cookies:[],localStorage:[],headers:{},userData:{}};let apiStateData;if("orig"===cookiesSource)stateData.cookies=cookies;else if("new"===cookiesSource)if(this.#taskContext.page)stateData.cookies=await this.#taskContext.page.cookies();else{if(!this.#taskContext.standaloneApiContext)throw new Error(`##templ ${this.#templateId} has no page when getting stateData of page`);apiStateData\|\|(apiStateData=await this.#taskContext.standaloneApiContext.stateData()),stateData.cookies=apiStateData.cookies}if("orig"===localStorageSource)stateData.localStorage=localStorage;else if("new"===localStorageSource)if(this.#taskContext.page)stateData.localStorage=await this.#taskContext.page.localStroage();else{if(!this.#taskContext.standaloneApiContext)throw new Error(`##templ ${this.#templateId} has no page when getting stateData of page`);apiStateData\|\|(apiStateData=await this.#taskContext.standaloneApiContext.stateData()),stateData.localStorage=apiStateData.localStorage}const interceptionData=this.#interceptionData;interceptionData[headersSource]&&(stateData.headers=Object.assign({},interceptionData[headersSource])),"orig"===userDataSource?stateData.userData=userData:"new"===userDataSource&&(stateData.userData=Object.assign({},this.#paras.userData)),this.#taskContext.newStateData=stateData}async _performGetstatedataInMisc(miscCfg){return await this.#performGetstatedataInMisc(miscCfg)}async#performScrollAction(actCfg){const page=this.#taskContext.page;if(!page)return this.#procErrname("cfginvalid","actionScroll","noPage"),!1;this.#validateCfgType(actCfg);const subType=this.#getCfgStrAttr(actCfg,"subtype");let height=["by","to"].includes(subType)?parseInt(this.#getCfgStrAttr(actCfg,"height")):0;const unitType=["by","to"].includes(subType)?this.#getCfgStrAttr(actCfg,"unit"):"none";let scrollHeight=0,unitStr="";"bodypct"===unitType?(unitStr=await page.windowMember(["document","body","scrollHeight"]),scrollHeight=parseInt(unitStr)):"screenpct"===unitType?unitStr=await page.windowMember(["screen","height"]):"windowpct"===unitType&&(unitStr=await page.windowMember(["innerHeight"]));const unitInt=unitStr?parseInt(unitStr):1;if(["bodypct","screenpct","windowpct"].includes(unitType)){if("number"!=typeof unitInt\|\|unitInt<=0)throw new Error("##te invalid unitInt in performScrollAction");height=Math.floor(heightunitInt/100)}let maxTimes="by"===subType?parseInt(this.#getCfgStrAttr(actCfg,"maxtimes")):0;const interval="by"===subType?parseInt(this.#getCfgStrAttr(actCfg,"interval")):0;if("by"===subType){if(0===maxTimes){if(scrollHeight<=0){const heightStr=await page.windowMember(["document","body","scrollHeight"]);scrollHeight=parseInt(heightStr)}maxTimes=Math.ceil(scrollHeight/height-1)}else maxTimes>200?maxTimes=200:maxTimes<0&&(maxTimes=10);for(let i=0;i<maxTimes;i++){await page.scrollBy(0,height),await(0,import_utils9.sleep)(interval);const eleCfgs=this.#getCfgChildrenOf(actCfg,"element");if(eleCfgs&&eleCfgs.length>0){const{elementSource:elementSource,elements:elements}=await this.#processElementCfgs(eleCfgs,1);if(this.#checkBrowserElementSource(elementSource,"scrollBy"),elements.length>0)break}}}else if("to"===subType)await page.scrollTo(0,height);else if("intoview"===subType){const eleCfgs=this.#getCfgChildrenOf(actCfg,"element",!0),{elementSource:elementSource,elements:elements}=await this.#processElementCfgs(eleCfgs,1);if(this.#checkBrowserElementSource(elementSource,"scrollIntoView"),elements.length<=0){const errName=this.#getCfgStrAttr(actCfg,"errname");return this.#procErrname(errName,"actionScrollIntoview","noElements"),!1}await elements[0].scrollIntoView()}return this.#addTaskCredits(actCfg),!0}async#selectOneOption(eleCfgs,selectType,selectVal,errName,tryTimes=1){if(eleCfgs.length<=0)throw new Error("##te No element cfg in select action");const{elementSource:elementSource,elements:elements}=await this.#processElementCfgs(eleCfgs,tryTimes);if(this.#checkBrowserElementSource(elementSource,"select"),elements.length<=0)return this.#procErrname(errName,"actionSelect","noElements"),!1;const options={type:"value"},items=selectVal.split(",");return"index"===selectType?(options.type,options.indexes=items.map(item=>parseInt(item))):"label"===selectType?(options.type,options.labels=items):"value"===selectType?(options.type,options.values=items):(0,import_utils9.unreachable)(selectType),await elements[0].select(options),!0}async#performSelectAction(actCfg){const page=this.#taskContext.page;if(!page)return this.#procErrname("cfginvalid","actionSelect","noPage"),!1;page.title,this.#validateCfgType(actCfg);const selectType=this.#getCfgStrAttr(actCfg,"selecttype"),selectVal=this.#getCfgStrAttr(actCfg,"selectval"),tryTimes=parseInt(this.#getCfgStrAttr(actCfg,"try")),errName=this.#getCfgStrAttr(actCfg,"errname"),eleCfgs=this.#getCfgChildrenOf(actCfg,"element",!0);await this.#selectOneOption(eleCfgs,selectType,selectVal,errName,tryTimes);const waitMsStr=this.#getCfgStrAttr(actCfg,"wait");return await this.#processWait(waitMsStr),this.#addTaskCredits(actCfg),!0}async#performSetvarAction(actCfg){this.#validateCfgType(actCfg);const subType=this.#getCfgStrAttr(actCfg,"subtype"),capSubtype=subType.slice(0,1).toUpperCase()+subType.slice(1),varName=this.#getCfgStrAttr(actCfg,"varname"),setKeys=this.#getCfgStrAttr(actCfg,"setkeys"),errName=this.#getCfgStrAttr(actCfg,"errname"),valErrName=this.#getCfgStrAttr(actCfg,"valerrname"),pattern=this.#getCfgStrAttr(actCfg,"pattern"),defaultVal=this.#getCfgStrAttr(actCfg,"defaultval");let targetFilename=this.#getCfgStrAttr(actCfg,"path"),content="";try{if(["element","get","templstr","subtask","ocr"].includes(subType)){content=(await this.#getEleOrTemplTransform(subType,actCfg)).content}else if("dbquery"===subType)content=await this.__processDbquery(actCfg,defaultVal);else{if("file"!==subType)throw new Error(`##te Invalid subtupe ${subType} in setvar action`);{const fileCfg=this.#getRequiredCfgChildOf(actCfg,"file");let sourceFilename=this.#getCfgStrAttr(fileCfg,"path");const encoding=this.#getCfgStrAttr(fileCfg,"encoding"),proxyFlag=this.#getCfgBoolAttr(actCfg,"proxy"),lcFilename=sourceFilename.toLocaleLowerCase();if(sourceFilename.includes("..")&&this.#procErrname("cfginvalid",`actionSetvar${capSubtype}`,"invalidSourceFilename"),lcFilename.startsWith("http://")\|\|lcFilename.startsWith("https://")){const proxyUrl=proxyFlag&&this.#taskContext.proxy?this.#taskContext.proxy.proxyUrl:"";content=(await lsdDownload(sourceFilename,proxyUrl)).toString(encoding)}else path5.isAbsolute(sourceFilename)?appConfig.cfg.inputFileDirPart&&!path5.dirname(sourceFilename).split(path5.sep).includes(appConfig.cfg.inputFileDirPart)&&"ignore"===errName&&this.#procErrname("normal",`actionSetvar${capSubtype}`,"notSecurePath"):sourceFilename=path5.join(appConfig.cfg.baseFileDir,sourceFilename),content=fs6.readFileSync(sourceFilename,{encoding:encoding})}}if("ignore"!==valErrName&&pattern){const flags=this.#getCfgStrAttr(actCfg,"flags"),re=this.#getRegExp(pattern,flags);re&&!content.match(re)&&this.#procErrname(valErrName,`actionSetvar${capSubtype}`,"validate")}if(!content&&defaultVal&&(content=defaultVal),varName&&this.#setNormalVariable(varName,content),setKeys&&content){const obj=(0,import_utils9.getJsonParseOfStr)(content);if(obj&&"object"==typeof obj){const keys=""===setKeys?Array.from(Object.keys(obj)):setKeys.split(",");for(const key of keys){let str="",val=obj[key];switch(typeof val){case"string":str=val;break;case"number":case"bigint":str=String(val);break;case"boolean":str=val?"1":"0";break;case"object":str=val?JSON.stringify(val):""}this.#setNormalVariable(key,str)}}}return targetFilename&&(targetFilename=this.#getAbsolutePath(targetFilename),fs6.writeFileSync(targetFilename,content)),!0}catch(err){return loginfo(err),err.message.includes("invalidTaskParas")?(this.#procErrname("parasinvalid","actionSetvar","invalidParasstr"),!1):(content="",this.#procErrname(errName,`actionSetvar${capSubtype}`,"psaException"),!1)}}async#performSubtaskAction(actCfg){this.#validateCfgType(actCfg);const subtasksStr=this.#getCfgStrAttr(actCfg,"subtasks"),sameAsParentFlag=this.#getCfgBoolAttr(actCfg,"sameasparent");if(subtasksStr&&subtasksStr.startsWith("[{"))try{const subtasks=JSON.parse(subtasksStr);if(Array.isArray(subtasks)&&subtasks.every(s=>s.tid>0&&"string"==typeof s.parasstr))for(const st of subtasks){if(this.#taskContext.subtasks.find(item=>item.tid===st.tid&&item.parasstr===st.parasstr)\|\|st.minlen>0&&st.parasstr.length<st.minlen)continue;const subtask={tid:st.tid,parasstr:st.parasstr};st.idx>0&&(subtask.idx=st.idx),sameAsParentFlag&&(subtask.sapFlag=!0),this.#taskContext.subtasks.push(subtask)}else subtasks.length>=1&&("string"==typeof subtasks[0].parasstr&&"number"==typeof subtasks[0].tid\|\|logerr(`##te invalid subtask: ${JSON.stringify(subtasks[0])} in performSubtaskAction`))}catch(err){logerr(err)}const subtaskCfgs=this.#getCfgChildrenOf(actCfg,"subtask");for(const subtaskCfg of subtaskCfgs){const templateId=parseInt(this.#getCfgStrAttr(subtaskCfg,"tid")),minLen=parseInt(this.#getCfgStrAttr(subtaskCfg,"minlen")),parasStr=this.#getCfgStrAttr(subtaskCfg,"parasstr"),idxStr=this.#getCfgStrAttr(subtaskCfg,"idx").trim(),idx=idxStr.length>0&&idxStr.match(/^[0-9]+$/)?parseInt(idxStr):0,errName=this.#getCfgStrAttr(subtaskCfg,"errname"),popuppageFlag=this.#getCfgBoolAttr(subtaskCfg,"popuppage");if(popuppageFlag&&!this.#taskContext.popupPage){const nopageErrName="ignore"===errName?"cfginvalid":errName;this.#procErrname(nopageErrName,"actionSubtask","noPopupPage")}if(parasStr.length>=minLen)if(popuppageFlag){const page=await this.#getSubtaskPage(popuppageFlag),subtaskResult=await this.#performOneTask(templateId,parasStr,this.#getSubtaskNetworkContext(page),"indSync");await this.#freeSubtaskPage(page),this.#taskContext.popupPage=null;const{credits:credits,execData:execData}=subtaskResult.taskData;credits<-1?this.#procErrname(errName,"actionSubtask","SyncIndepFailed"):(this.#taskContext.subtaskResults.push(subtaskResult),logdbg(`##te indsyncsubtask t${templateId} p${parasStr} c${credits} data: ${JSON.stringify(execData)}`))}else{if(!this.#taskContext.subtasks.find(item=>item.tid===templateId&&item.parasstr===parasStr)){const subtask={tid:templateId,parasstr:parasStr};idx>0&&(subtask.idx=idx),sameAsParentFlag&&(subtask.sapFlag=!0),this.#taskContext.subtasks.push(subtask)}}}return!0}async#performWaitAction(actCfg){this.#validateCfgType(actCfg);const subType=this.#getCfgStrAttr(actCfg,"subtype"),errName=this.#getCfgStrAttr(actCfg,"errname"),waitMsStr=this.#getCfgStrAttr(actCfg,"wait");if("sleep"===subType){const waitTime1InMs=parseInt(this.#getCfgStrAttr(actCfg,"minms")),waitTime2InMs=parseInt(this.#getCfgStrAttr(actCfg,"maxms")),waitTime=(0,import_utils9.getRandomInt)(waitTime1InMs,waitTime2InMs);return await(0,import_utils9.sleep)(waitTime),await this.#processWait(waitMsStr),!0}const page=this.#taskContext.page;if("element"===subType){if(!page)return this.#procErrname("cfginvalid","actionWaitElement","noPage"),!1;const eleCfgs=this.#getCfgChildrenOf(actCfg,"element",!0);eleCfgs.length<=0&&this.#procErrname("cfginvalid","actionWaitElement","noElements");const selector=this.#getCfgStrAttr(eleCfgs[0],"loc"),timeout=parseInt(this.#getCfgStrAttr(actCfg,"timeout")),state=this.#getCfgStrAttr(actCfg,"state");try{await page.waitForElement(selector,{timeout:timeout,state:state})}catch(err){logerr(err),this.#procErrname(errName,"actionWaitElement","pwaException")}}else if("navigation"===subType){if(!page)return this.#procErrname("cfginvalid","actionWaitNavigation","noPage"),!1;const timeout=parseInt(this.#getCfgStrAttr(actCfg,"timeout")),waitUntil=this.#getCfgStrAttr(actCfg,"waituntil"),urlStr=this.#getCfgStrAttr(actCfg,"url");waitUntil\|\|this.#procErrname("cfginvalid","actionWaitNavigation","invlidWaitUntil");try{if(urlStr){const url=new RegExp(urlStr);await page.waitForNavigation({url:url,timeout:timeout,waitUntil:waitUntil})}else await page.waitForNavigation({timeout:timeout,waitUntil:waitUntil})}catch(err){logerr(err),this.#procErrname(errName,"actionWaitNavigation","pwaException")}}else this.#procErrname("cfginvalid","actionWait","invalidSubtype");return await this.#processWait(waitMsStr),!0}async#performActions(actCfgs){if(!Array.isArray(actCfgs))throw new Error("##te Ivalid action configs(actions)");for(const actCfg of actCfgs)try{const actType=this.#getCfgStrAttr(actCfg,"type");if(this.#skipActions.includes(actType)\|\|this.#skipActions.includes("operate")&&appConfig.OPERATE_ACTIONS.includes(actType))continue;if(await this.#actionPerformers[actType].call(this,actCfg),this.#taskContext.continueFlag\|\|this.#taskContext.breakFlag)break}catch(err){throw this.#taskContext.credits>=0&&(loginfo(`##te tId${this.#templateId}, aCfg: ${JSON.stringify(actCfg).slice(0,200)}`),loginfo(`##te err.stack: ${err.stack}`)),err}}async#execTaskActions(){try{const page=this.#taskContext.page,captchaTypes=this.#captchaOptions.captchaTypes;page&&captchaTypes.length>0&&await injectScripts(page,captchaTypes);const subactions=this.#getCfgChildrenOf(this.#actionConfigs,"action");await this.#performActions(subactions),this.emit(appConfig.EVENT_TYPES.taskComplete,{code:0,message:"ok",data:null})}catch(err){this.emit(appConfig.EVENT_TYPES.taskComplete,{code:1001,message:"failed",data:err})}}#getNormalVariables(){const variables={};for(const[key,value]of Object.entries(this.#paras))"string"==typeof key&&"string"==typeof value&&(variables[key]=value);return variables}#addListeners(){const page=this.#taskContext.page;if(page){page.on(appConfig.EVENT_TYPES.lsdPageClose,()=>{this.emit(appConfig.EVENT_TYPES.taskComplete,{code:1005,message:"pageClosed",data:null})});const maxExecutionDuration=this.#template.maxExecutionDuration;maxExecutionDuration>0&&setTimeout(()=>this.emit(appConfig.EVENT_TYPES.taskComplete,{code:1009,message:"timeout",data:null}),1e3maxExecutionDuration),page.pageInfo().relatedId>0&&page.on(appConfig.EVENT_TYPES.lsdPagePopup,async eventData=>{loginfo("##tmp LsdPage received popup event: ",eventData),this.emit(appConfig.EVENT_TYPES.lsdScraperPopup,{code:0,message:"ok",data:eventData})})}}#removeListeners(){const page=this.#taskContext.page;page&&(page.removeAllListeners(appConfig.EVENT_TYPES.lsdPagePopup),page.removeAllListeners(appConfig.EVENT_TYPES.lsdPageClose)),this.removeAllListeners(appConfig.EVENT_TYPES.taskComplete)}async run(){const taskData={templateId:this.#templateId,parasStr:this.#parasStr,credits:0,execData:{},subtasks:[]},page=this.#taskContext.page;let subtaskDatas;loginfo(page?`##tmp -- start to run tId${this.#templateId} kId${this.#taskId} page ${page.id()}`:`##tmp -- start to run tId${this.#templateId} kId${this.#taskId} no page`);try{this.#addListeners(),this.#execTaskActions();const[response]=await(0,import_node_events2.once)(this,appConfig.EVENT_TYPES.taskComplete);if(0===response.code);else if(1001===response.code){if(response.data instanceof Error)throw response.data;logerr(response.data)}else 1005===response.code?this.#procErrname("other","task","pageClosed"):1009===response.code&&this.#procErrname("tetimeout","task","teTimeout");page&&this.#lastUrl!==page.url()&&loginfo(`##te kId${this.#taskId} endUrl: ${page.url()}`);const subtaskCnt=this.#taskContext.subtasks.length;subtaskCnt>0&&loginfo(`##te kId${this.#taskId} has ${subtaskCnt} subtasks`);for(let tablename of Object.keys(this.#taskContext.execData)){const recordsCnt=this.#taskContext.execData[tablename].length;loginfo(`##te kId${this.#taskId} tablename ${tablename} has ${recordsCnt}`)}return this.#removeListeners(),loginfo(`##tmp teSucceeded kId${this.#taskId} tId${this.#templateId}`),taskData.execData=this.#taskContext.execData,taskData.subtasks=this.#taskContext.subtasks,taskData.credits=this.#taskContext.credits,this.#taskContext.subtaskResults.length>0&&(subtaskDatas=this.#taskContext.subtaskResults.map(tr=>tr.taskData)),await this.__whenTaskEndFun(),taskData.credits<-30&&logwarn(`##te taskId ${this.#taskId} credits ${taskData.credits}`),{taskData:taskData,subtaskDatas:subtaskDatas,newStateData:this.#taskContext.newStateData?this.#taskContext.newStateData:void 0}}catch(err){taskData.execData=this.#taskContext.execData,taskData.subtasks=this.#taskContext.subtasks,taskData.credits=this.#taskContext.credits,this.#taskContext.subtaskResults.length>0&&(subtaskDatas=this.#taskContext.subtaskResults.map(tr=>tr.taskData));const newStateData=this.#taskContext.newStateData?this.#taskContext.newStateData:void 0,stack="string"==typeof err?.stack?err.stack:"unkown",misc={taskId:this.#taskId,message:"unknown",stack:stack,variables:this.#getNormalVariables()};if(this.#removeListeners(),await this.__whenTaskEndFun(),this.#taskContext.credits>=0)logerr(`##te Exeception in ScraperTask.run: kId ${this.#taskId}`),logerr(err);else{if(-1===this.#taskContext.credits)return{taskData:taskData,subtaskDatas:subtaskDatas,newStateData:newStateData,misc:misc};if(this.#taskContext.credits>=-30)return{taskData:taskData,subtaskDatas:subtaskDatas,newStateData:newStateData,misc:misc};if(2===this.#taskId&&"check"===this.#parasStr&&page){logdbg(`##cap loginCheckFailed capabilityId ${0} credits: ${this.#taskContext.credits}`)}else 11===this.#taskId\|\|-69===this.#taskContext.credits\|\|logwarn(`##te taskExec run execption: templateId ${this.#templateId}, paras ${this.#parasStr}, credits ${this.#taskContext.credits}`)}return logwarn(`##tmp taskExec run execption: templateId ${this.#templateId}, paras ${this.#parasStr}, credits ${this.#taskContext.credits}`),taskData.credits=this.#taskContext.credits>=0?-99:this.#taskContext.credits,{taskData:taskData,subtaskDatas:subtaskDatas,newStateData:newStateData,misc:misc}}}async __performMiscAction(actCfg){this.#validateCfgType(actCfg);const miscCfgs=this.#getCfgChildrenOf(actCfg,"misc");if(miscCfgs.length<=0)return logerr(`##templ ${this.#templateId} invalid template ${this.#templateId} in performMiscAction`),!1;for(const miscCfg of miscCfgs){const type=this.#getCfgStrAttr(miscCfg,"type");switch(type){case"delsyncdbdata":case"extractdata":this.#procErrname("cfginvlid","notSupported",type);break;case"getstatedata":await this.#performGetstatedataInMisc(miscCfg);break;default:(0,import_utils9.unreachable)(type)}}return!0}async __processDbquery(actCfg,defaultVal){return(actCfg\|\|defaultVal)&&this.#procErrname("cfginvlid","notSupported","dbquery"),""}async __processBrowserOcrCfg(ocrCfg,$browserEles){return(ocrCfg\|\|$browserEles)&&this.#procErrname("cfginvlid","notSupported","ocr"),""}async __decodeFontInContent(content,decodefontsvgCfg,decodefontttfCfg){return(content\|\|decodefontsvgCfg\|\|decodefontttfCfg)&&this.#procErrname("cfginvlid","notSupported","decodeFont"),""}async __whenTaskEndFun(){return!0}},path6=__toESM(require("path"),1),fsp2=__toESM(require("fs/promises"),1),import_utils10=require("@letsscrapedata/utils");async function moveDataFiles(){try{const exportsDir=appConfig.cfg.exportsDir;if(!exportsDir\|\|!(0,import_utils10.isDirectory)(exportsDir))return!1;const filenames=await(0,import_utils10.filesInDir)(exportsDir,"general","dat_");if(0===filenames.length)return!0;const dateNum=(0,import_utils10.getLocalDateNumber)(),oldDataDir=path6.join(exportsDir,String(dateNum));await(0,import_utils10.fsCheckOrCreateDir)(oldDataDir,!0,!0);for(const filename of filenames){const oldFilename=path6.join(exportsDir,filename),newFilename=path6.join(oldDataDir,filename);await fsp2.rename(oldFilename,newFilename)}return!0}catch(err){return logerr(err),!1}}async function writeDataIntoOneFile(filename,records,columnSeperator){if(records.length<=0)return!1;const dataFileFormat=appConfig.cfg.dataFileFormat;if("jsonl"!==dataFileFormat&&!await(0,import_utils10.isFile)(filename)){const keys=Array.from(Object.keys(records[0]));await fsp2.writeFile(filename,keys.join(columnSeperator).concat("\n"))}const content="jsonl"===dataFileFormat?records.map(r=>JSON.stringify(r)).join("\n"):records.map(r=>Object.values(r).join(columnSeperator)).join("\n");return await fsp2.appendFile(filename,content.concat("\n")),!0}async function saveExecDataOfTask(execData){try{const{exportsDir:exportsDir,dataFileFormat:dataFileFormat,columnSeperator:columnSeperator}=appConfig.cfg;if(!(exportsDir&&(0,import_utils10.isDirectory)(exportsDir)&&dataFileFormat&&columnSeperator))return!1;for(const[datatableName,records]of Object.entries(execData)){const filename=path6.join(exportsDir,`${datatableName}.${dataFileFormat}`);await writeDataIntoOneFile(filename,records,columnSeperator)}return!0}catch(err){return logerr(err),!1}}async function performOneTask(templateId,parasStr,taskNetworkContext,taskType="indAsync",xmlStr="",taskId=0){try{const parsedTemplateExt=await TemplateManagerInScraper.getTemplateConfig(templateId,xmlStr),{template:template,actionConfigs:actionConfigs,fontsConfig:fontsConfig,myfunScripts:myfunScripts,paraCfgs:paraCfgs,attrsInXml:attrsInXml,captchaTypes:captchaTypes}=parsedTemplateExt,templateConfig={template:template,actionConfigs:actionConfigs,fontsConfig:fontsConfig,myfunScripts:myfunScripts},{skipActions:skipActions="",extractColumns:extractColumns="",defaultParasStr:defaultParasStr}=attrsInXml\|\|{},taskParasStr=parasStr\|\|(appConfig.cfg.useParasstrInXmlIfNeeded&&defaultParasStr?defaultParasStr:""),inParas=TaskParser.getInParas(taskParasStr,paraCfgs),taskOptions=1===taskId&&attrsInXml?{skipActions:skipActions?skipActions.split(","):[],extractColumns:extractColumns?extractColumns.split(","):[]}:{};taskOptions.captchaOptions={captchaTypes:captchaTypes};const scraper2=new TaskScraper(templateConfig,inParas,taskNetworkContext,taskOptions),taskResult=await scraper2.run();if("indSync"===taskType\|\|"indAsync"===taskType)if(taskResult.taskData.credits>=-1)loginfo(`Task are completed: templateId ${templateId}, parasStr ${parasStr}`),await saveExecDataOfTask(taskResult.taskData.execData);else if("indSync"===taskType){const execData={dat_0000000000000001:[{c01:String(templateId),c02:parasStr,c03:String(taskResult.taskData.credits),c04:(0,import_utils11.getLocalDateString)()}]};await saveExecDataOfTask(execData)}return taskResult}catch(err){const misc={taskId:0,message:err.message,stack:err.stack,variables:{}};if("indSync"===taskType){const execData={dat_0000000000000001:[{c01:String(templateId),c02:parasStr,c03:"-1",c04:(0,import_utils11.getLocalDateString)()}]};await saveExecDataOfTask(execData)}return{taskData:{templateId:templateId,parasStr:parasStr,credits:0,execData:{},subtasks:[]},subtaskDatas:[],misc:misc}}}appConfig.performOneTask=performOneTask;var path8=__toESM(require("path"),1),import_utils12=require("@letsscrapedata/utils"),import_controller4=require("@letsscrapedata/controller"),BrowserManagerInScraper=class{#config;#controller;#browsers;#browserContextInfos;#getMaxConcurrency(templateId){const cfg=this.#config.templateParas.find(tmc=>tmc.templateId===templateId);return cfg?.maxConncurrency?cfg.maxConncurrency:this.#config.maxConcurrency}async#addBrowserContext(browserContext,proxy){const standaloneApiContext=await import_controller4.controller.newApiContext({proxy:proxy}),browserApiContext="playwright"===browserContext.browser().browserControllerType()?browserContext.apiContext():null;this.#browserContextInfos.push({proxy:proxy,browserContext:browserContext,browserApiContext:browserApiContext,standaloneApiContext:standaloneApiContext,templateMap:new Map})}constructor(){const{useStealthPlugin:useStealthPlugin,browserControllerType:browserControllerType,browserType:browserType,lsdLaunchOptions:lsdLaunchOptions,lsdConnectOptions:lsdConnectOptions,maxConcurrency:maxConcurrency,templateParas:templateParas}=appConfig.cfg,browserConfigs=Array.isArray(appConfig.cfg.browserConfigs)&&appConfig.cfg.browserConfigs.length>0?appConfig.cfg.browserConfigs:[{browserControllerType:browserControllerType,browserType:browserType,proxyUrl:""}];this.#config={useStealthPlugin:useStealthPlugin,browserControllerType:browserControllerType,browserType:browserType,lsdLaunchOptions:lsdLaunchOptions,lsdConnectOptions:lsdConnectOptions,browserConfigs:browserConfigs,maxConcurrency:maxConcurrency,templateParas:templateParas},this.#controller=import_controller4.controller,["playwright","puppeteer"].includes(browserControllerType),this.#browsers=[],this.#browserContextInfos=[]}async init(){const{browserControllerType:defaultBrowserControllerType,browserType:defaultBrowserType,lsdLaunchOptions:lsdLaunchOptions,lsdConnectOptions:lsdConnectOptions}=this.#config;for(const browserConfig of this.#config.browserConfigs){const{browserControllerType:browserControllerType=defaultBrowserControllerType,browserType:browserType=defaultBrowserType,browserUrl:browserUrl,proxyUrl:proxyUrl=""}=browserConfig,proxy=this.#convertProxy(proxyUrl);if(browserUrl){const options=Object.assign({},lsdConnectOptions,{browserUrl:browserUrl}),browser=await this.#controller.connect(browserControllerType,browserType,options);this.#browsers.push(browser);for(const bc of browser.browserContexts())await this.#addBrowserContext(bc,proxy)}else if("string"==typeof proxyUrl){const browserProxy=proxyUrl?proxy:void 0,options=Object.assign({},lsdLaunchOptions,{proxy:browserProxy}),browser=await this.#controller.launch(browserControllerType,browserType,options);this.#browsers.push(browser);for(const bc of browser.browserContexts())await this.#addBrowserContext(bc,proxy)}}return await(0,import_utils12.sleep)(2e3),!0}#convertProxy(proxyUrl,proxyId="",username="",password=""){return{proxyUrl:proxyUrl,proxyId:proxyId\|\|proxyUrl,username:username,password:password}}async getTaskNetworkContext(templateId=0){if(0===this.#browserContextInfos.length)throw new Error("please call init() first to launch or connect browsers.");let templateMapValue;const randomIdxes=(0,import_utils12.getRandomArrayIdx)(this.#browserContextInfos.length);for(const idx of randomIdxes){const browserContextInfo=this.#browserContextInfos[idx],{proxy:proxy,browserContext:browserContext,browserApiContext:browserApiContext,standaloneApiContext:standaloneApiContext,templateMap:templateMap}=browserContextInfo;if(templateId>0&&(templateMapValue=templateMap.get(templateId),templateMapValue\|\|(templateMapValue={templateId:templateId,maxConcurrency:this.#getMaxConcurrency(templateId),running:0},templateMap.set(templateId,templateMapValue)),templateMapValue.maxConcurrency<=templateMapValue.running))continue;if(!browserContext.hasFreePage())continue;const page=await browserContext.getPage();return page?(templateId>0&&templateMapValue&&templateMapValue.running++,page.setPageInfo({relatedId:templateId}),{proxy:proxy,page:page,browserApiContext:browserApiContext,standaloneApiContext:standaloneApiContext}):null}return null}async freePage(page){const templateId=page.pageInfo().relatedId;if(templateId>0){const browserContextId=page.browserContext().id(),browserContextInfo=this.#browserContextInfos.find(bci=>bci.browserContext.id()===browserContextId);if(!browserContextInfo)throw new Error("No browserContext has the page to free.");const templateMapInfo=browserContextInfo.templateMap.get(templateId);if(!templateMapInfo)throw new Error(`No templateMapInfo has templateId ${templateId} in ${browserContextId}.`);templateMapInfo.running--}return await page.free(),!0}},fsp3=__toESM(require("fs/promises"),1),path7=__toESM(require("path"),1),import_utils13=require("@letsscrapedata/utils"),SchedulerInScraper=class _SchedulerInScraper{#taskQueueMap;#taskDIr;#exitWhenCompleted;#browserManager;runningTaskSet;static#newLineChar="\n";static#tryingFlag=!1;constructor(browserManager){this.#taskQueueMap=new Map,this.#taskDIr=appConfig.cfg.taskDir,this.#exitWhenCompleted=appConfig.cfg.exitWhenCompleted,this.#browserManager=browserManager,this.runningTaskSet=new Set}async#readParasStrsFromeFile(filename,deleteFile=!1){try{const lines=(await fsp3.readFile(filename,"utf8")).split(_SchedulerInScraper.#newLineChar),parasStrSet=new Set(lines);return parasStrSet.size>1&&parasStrSet.has("")&&parasStrSet.delete(""),deleteFile&&await fsp3.unlink(filename),Array.from(parasStrSet.values())}catch(err){return[]}}static#templateLockMap=new Map;async#writeTasksIntoFile(parasStrs,templateId,failed=!1){if(_SchedulerInScraper.#templateLockMap.get(templateId)){for(let i=0;i<50&&(await(0,import_utils13.sleep)(200),_SchedulerInScraper.#templateLockMap.get(templateId));i++);if(_SchedulerInScraper.#templateLockMap.get(templateId))return!1}_SchedulerInScraper.#templateLockMap.set(templateId,!0);try{if(failed){const filename=path7.join(this.#taskDIr,`failedtempl${templateId}.txt`);return await fsp3.appendFile(filename,parasStrs.join(_SchedulerInScraper.#newLineChar).concat(_SchedulerInScraper.#newLineChar)),!0}{const filename=path7.join(this.#taskDIr,`tid${templateId}.txt`);return parasStrs.length>0?await fsp3.writeFile(filename,parasStrs.join(_SchedulerInScraper.#newLineChar)):await(0,import_utils13.isFile)(filename)&&await fsp3.unlink(filename),_SchedulerInScraper.#templateLockMap.delete(templateId),!0}}catch(err){return logwarn(`Failed to write task queue of tempatedId ${templateId} into file`),_SchedulerInScraper.#templateLockMap.delete(templateId),!1}}async#loadFailedTasks(){const taskDir=appConfig.cfg.taskDir;if(!taskDir\|\|!(0,import_utils13.isDirectory)(taskDir))return!1;const filenames=await(0,import_utils13.filesInDir)(taskDir,"general","failedtempl","txt");for(const filename of filenames){const templateId=parseInt(filename.slice(11,-4)),parasStrs=await this.#readParasStrsFromeFile(path7.join(taskDir,filename),!0);if(parasStrs.length>0){loginfo(`load unfinished ${parasStrs.length} tasks of templateId ${templateId}`);const queue=this.#taskQueueMap.get(templateId);queue?parasStrs.forEach(parasStr=>queue.push(parasStr)):this.#taskQueueMap.set(templateId,parasStrs)}}return!0}async#loadUnfinishedTasks(loadFailedTasks=!1){const taskDir=appConfig.cfg.taskDir;if(!taskDir\|\|!(0,import_utils13.isDirectory)(taskDir))return!1;const filenames=await(0,import_utils13.filesInDir)(taskDir,"general","templ","txt");for(const filename of filenames){const templateId=parseInt(filename.slice(5,-4)),parasStrs=await this.#readParasStrsFromeFile(path7.join(taskDir,filename));if(parasStrs.length>0){loginfo(`load unfinished ${parasStrs.length} tasks of templateId ${templateId}`);const queue=this.#taskQueueMap.get(templateId);queue?parasStrs.forEach(parasStr=>queue.push(parasStr)):this.#taskQueueMap.set(templateId,parasStrs)}}return loadFailedTasks&&await this.#loadFailedTasks(),!0}async addNewTask(newTask,tryToExecute){const{tid:tid,parasstrs:parasstrs}=newTask;if(tid<=0\|\|0===parasstrs.length)return!1;let queue=this.#taskQueueMap.get(tid);return queue\|\|(queue=[],this.#taskQueueMap.set(tid,queue)),parasstrs.forEach(p=>{queue.includes(p)\|\|queue.push(p)}),await this.#writeTasksIntoFile(queue,tid),tryToExecute&&await this.tryToExecuteTasks([tid]),!0}async init(newTasks=[]){appConfig.cfg.moveDataWhenStart&&await moveDataFiles();const loadFailedTasks=appConfig.cfg.loadFailedTasksInterval>0;appConfig.cfg.loadUnfinishedTasks&&await this.#loadUnfinishedTasks(loadFailedTasks);for(const task of newTasks){const{tid:tid,parasstrs:parasstrs}=task;this.addNewTask({tid:tid,parasstrs:parasstrs},!1)}return await this.tryToExecuteTasks(),setInterval(this.tryToExecuteTasks.bind(this),6e4),loadFailedTasks&&setInterval(this.#loadFailedTasks.bind(this),6e4appConfig.cfg.loadFailedTasksInterval),!0}#getTaskIdStr(templateId,parasStr){return`${templateId}@@${parasStr}`}async#tryToExecuteOneTask(templateId,parasStr,taskNetworkContext){const taskResult=await performOneTask(templateId,parasStr,taskNetworkContext,"indAsync","",0),{credits:credits,subtasks:subtasks}=taskResult.taskData;let templateIdSet=new Set;const parasStrs=this.#taskQueueMap.get(templateId);if(parasStrs){const idx=parasStrs.findIndex(ps=>ps===parasStr);if(idx<0?logerr(`Task queue of templateId ${templateId} has no parasStr: ${parasStr}`):parasStrs.splice(idx,1),credits>=-1){if(0===parasStrs.length&&this.#taskQueueMap.delete(templateId),subtasks.length>0){templateIdSet=new Set(subtasks.map(s=>s.tid));for(const tid of templateIdSet){const parasstrs=subtasks.filter(s=>s.tid===tid).map(s=>s.parasstr);await this.addNewTask({tid:tid,parasstrs:parasstrs},!1)}}await this.#writeTasksIntoFile(parasStrs,templateId)}else await this.#writeTasksIntoFile([parasStr],templateId,!0);this.runningTaskSet.delete(this.#getTaskIdStr(templateId,parasStr))}else logerr(`No task queue of templateId ${templateId}`);return taskNetworkContext.page&&await this.#browserManager.freePage(taskNetworkContext.page),appConfig.cfg.minMiliseconds>0&&await(0,import_utils13.sleep)(appConfig.cfg.minMiliseconds),await this.tryToExecuteTasks([templateId],1),templateIdSet.size>0&&await this.tryToExecuteTasks(),!0}async tryToExecuteTasks(templateIds=[],maxNum=0){if(loginfo(`---- tryToExecuteTasks templateIds ${templateIds.join(",")} : `,this.#taskQueueMap),_SchedulerInScraper.#tryingFlag)return logdbg("---- another tryToExecuteTasks is running now"),!0;this.#exitWhenCompleted&&0===this.#taskQueueMap.size&&process.exit(0),_SchedulerInScraper.#tryingFlag=!0;try{let launchedNum=0;0===templateIds.length&&(templateIds=Array.from(this.#taskQueueMap.keys())),logdbg(`templateIds: ${templateIds.join(",")}`);for(const templateId of templateIds){const parasStrs=this.#taskQueueMap.get(templateId);if(!parasStrs)break;const len=parasStrs.length;for(let i=0;i<len;i++){const parasStr=parasStrs[i],taskIdStr=this.#getTaskIdStr(templateId,parasStr);if(this.runningTaskSet.has(taskIdStr))continue;const taskNetworkContext=await this.#browserManager.getTaskNetworkContext(templateId);if(!taskNetworkContext){logdbg(`No page to execute ${taskIdStr}`);break}if(this.runningTaskSet.add(taskIdStr),this.#tryToExecuteOneTask(templateId,parasStr,taskNetworkContext),launchedNum++,maxNum>0&&launchedNum>=maxNum)return _SchedulerInScraper.#tryingFlag=!1,!0}}return _SchedulerInScraper.#tryingFlag=!1,!0}catch(err){return logdbg(err),_SchedulerInScraper.#tryingFlag=!1,!1}}},import_utils15=require("@letsscrapedata/utils"),import_events=__toESM(require("events"),1),import_utils14=require("@letsscrapedata/utils"),TwoCaptcha=class{#clientKey;#status;#pollingInterval;#timeout;#callbackUrl;#supportedCaptchaTypes;#eventBus;async#fetch(url,body,method="POST"){try{const options={method:method,headers:{"Content-Type":"application/json"},body:JSON.stringify(body)},response=await fetch(url,options);if(!response.ok)return logerr(`##captcha failed to fetch, url ${url}`),null;return await response.json()}catch(err){return logerr(err),null}}async#createTask(task){const body={clientKey:this.#clientKey,task:task,softId:0};"string"==typeof task.lang&&task.lang&&(body.languagePool=task.lang,delete task.lang);const res=await this.#fetch("https://api.2captcha.com/createTask",body);return 10===res.errorId?this.setStatus("nobalance"):[11,21,55].includes(res.errorId)&&this.setStatus("disabled"),res}async#getResult(taskId){const body={clientKey:this.#clientKey,taskId:taskId};return await this.#fetch("https://api.2captcha.com/getTaskResult",body)}async#getBalance(){const body={clientKey:this.#clientKey},res=await this.#fetch("https://api.2captcha.com/getBalance",body);return res?.balance>=0?res.balance:-1}#getEventName(taskId){return`event${taskId}`}#getMinWaitMs(_captchaType){return 5e3}async#executeTask(task,captchaType){const response={errorCode:"SUCCEEDED",taskId:0,cost:0,ip:"",createTime:(0,import_utils14.getCurrentUnixTime)(),endTime:0,solveCount:0},{errorId:errorId,taskId:taskId}=await this.#createTask(task);if(errorId>0\|\|taskId<=0)return response.errorCode="ERROR_OTHER",taskId>0&&(response.taskId=taskId),response;response.taskId=taskId;const eventName=this.#getEventName(taskId);await(0,import_utils14.sleep)(this.#getMinWaitMs(captchaType));let interval=0;this.#callbackUrl\|\|(interval=setInterval(async()=>{const res=await this.#getResult(taskId);0!==res?.errorId?this.#eventBus.emit(eventName,{code:1001,message:"failed",data:res}):0===res.errorId&&"ready"===res.status&&this.#eventBus.emit(eventName,{code:0,message:"succeeded",data:res})},this.#pollingInterval)),setTimeout(()=>this.#eventBus.emit(eventName,{code:1009,message:"timeout",data:null}),this.#timeout);const[eventData]=await(0,import_events.once)(this.#eventBus,eventName);interval&&clearInterval(interval),response.endTime=(0,import_utils14.getCurrentUnixTime)();const{code:code,data:data}=eventData;if(0===code){if(!data\|\|0!==data.errorId)throw new Error("##captcha invalid data in getResult");const{cost:cost,ip:ip,createTime:createTime,endTime:endTime,solveCount:solveCount,solution:solution}=data;cost&&(response.cost=Number(cost)),ip&&(response.ip=ip),createTime&&createTime>0&&(createTime-response.createTime>5&&loginfo("##captcha task createTime difference: "+(createTime-response.createTime)),response.createTime=createTime),endTime&&endTime>0&&(response.endTime-endTime>2*this.#pollingInterval&&loginfo("##captcha task endTime difference: "+(response.endTime-endTime)),response.endTime=endTime),solveCount&&solveCount>0&&(response.solveCount=solveCount),solution&&(response.solution=solution)}else if(1001===code)response.errorCode=data?.errorCode?data.errorCode:"ERROR_OTHER";else{if(1009!==code)throw new Error(`##captcha unreachable at ${(new Error).stack}`);response.errorCode="ERROR_TIMEOUT"}return response}#convertTask(origTask,taskName,proxy){let type=proxy?taskName:`${taskName}Proxyless`;return proxy?Object.assign({},origTask,{type:type},proxy):Object.assign({},origTask,{type:type})}#convertGeetestTask(origTask,proxy){const{version:version,websiteURL:websiteURL,gt:gt,challenge:challenge,geetestApiServerSubdomain:geetestApiServerSubdomain,userAgent:userAgent,captchaId:captchaId}=origTask,task={type:proxy?"GeeTestTask":"GeeTestTaskProxyless",version:version,websiteURL:websiteURL};return gt&&(task.gt=gt),challenge&&(task.challenge=challenge),geetestApiServerSubdomain&&(task.geetestApiServerSubdomainapiDomain=geetestApiServerSubdomain),captchaId&&(task.initParameters={captcha_id:captchaId}),userAgent&&(task.userAgent=userAgent),task}#convertRecaptchaTask(origTask,proxy){const{version:version,websiteKey:websiteKey,websiteURL:websiteURL,isEnterprise:isEnterprise,isInvisible:isInvisible,s:s,action:action,minScore:minScore,userAgent:userAgent,cookies:cookies,apiDomain:apiDomain}=origTask;let type="RecaptchaV3TaskProxyless";if(2===version){const taskName=isEnterprise?"RecaptchaV2EnterpriseTask":"RecaptchaV2Task";type=proxy?taskName:`${taskName}Proxyless`}const task={type:type,websiteURL:websiteURL,websiteKey:websiteKey};return apiDomain&&(task.apiDomain=apiDomain),task.isInvisible=isInvisible,s&&(isEnterprise?task.enterprisePayload=s:task.recaptchaDataSValue=s),userAgent&&(task.userAgent=userAgent),cookies&&(task.cookies=cookies),task.isEnterprise=isEnterprise,minScore&&(task.minScore=minScore),action&&(task.pageAction=action),task}#convertTurnstileTask(origTask,proxy){let type=proxy?"TurnstileTask":"TurnstileTaskProxyless";const{websiteKey:websiteKey,websiteURL:websiteURL,action:action,cData:cData,chlPageData:chlPageData}=origTask;return action?{type:type,websiteKey:websiteKey,websiteURL:websiteURL,action:action,data:cData,pagedata:chlPageData}:{type:type,websiteKey:websiteKey,websiteURL:websiteURL}}#convertTextTask(origTask){const{images:images,comment:comment,lang:lang,math:math=!1,maxLength:maxLength=0,minLength:minLength=0,question:question=!1,space:space=!1,textType:textType="any"}=origTask;if(question)return{type:"TextCaptchaTask",comment:comment,lang:lang};{let numeric=0;"numbers"===textType?numeric=1:"letters"===textType&&(numeric=2);const task={type:"ImageToTextTask",body:images[0].toString("base64"),case:!!origTask.case,comment:comment,math:math,maxLength:maxLength,minLength:minLength,numeric:numeric,phrase:space,lang:lang};return images.length>1&&(task.imgInstructions=images[1].toString("base64")),task}}#convertCoordinateTask(origTask){const{images:images,comment:comment,lang:lang,maxClicks:maxClicks=0,minClicks:minClicks=1}=origTask,task={type:"CoordinatesTask",body:images[0].toString("base64"),comment:comment,lang:lang,minClicks:minClicks};return maxClicks>0&&maxClicks>=minClicks&&(task.maxClicks=maxClicks),images.length>1&&(task.imgInstructions=images[1].toString("base64")),task}constructor(options){const{clientKey:clientKey,callbackUrl:callbackUrl="",pollingInterval:pollingInterval=5e3,timeout:timeout=15e4}=options;if(!clientKey)throw new Error("Invalid options: no clientKey");this.#clientKey=clientKey,this.#status="normal",this.#pollingInterval=pollingInterval>=2e3?pollingInterval:2e3,this.#timeout=timeout>=3e4?timeout:3e4,this.#callbackUrl=callbackUrl\|\|"",callbackUrl&&loginfo(this.#callbackUrl),this.#supportedCaptchaTypes=["amazon","funcaptcha","geetest","keycaptcha","mtcaptcha","recaptcha","turnstile","text","coordinate","grid","slider","rotation"],this.#eventBus=new import_events.default}canSolveCaptcha(captchaType){return this.#supportedCaptchaTypes.includes(captchaType)}async getBalance(){return await this.#getBalance()}name(){return"2captcha"}sendResult(result){if(!result\|\|"object"!=typeof result)return!1;const{taskId:taskId,errorId:errorId}=result;if(taskId>0){const eventName=this.#getEventName(result.taskId);return 0===errorId?this.#eventBus.emit(eventName,{code:0,message:"succeeded",data:result}):0===errorId&&this.#eventBus.emit(eventName,{code:1001,message:"failed",data:result}),!0}return!1}status(){return this.#status}setStatus(newStatus){switch(newStatus){case"normal":case"disabled":case"nobalance":this.#status=newStatus;break;default:(0,import_utils14.unreachable)(newStatus)}return!0}async amazon(task,proxy){return await this.#executeTask(this.#convertTask(task,"AmazonTask",proxy),"amazon")}async funcaptcha(task,proxy){return await this.#executeTask(this.#convertTask(task,"FunCaptchaTask",proxy),"funcaptcha")}async geetest(task,proxy){return await this.#executeTask(this.#convertGeetestTask(task,proxy),"geetest")}async keycaptcha(task,proxy){return await this.#executeTask(this.#convertTask(task,"KeyCaptchaTask",proxy),"keycaptcha")}async mtcaptcha(task,proxy){return await this.#executeTask(this.#convertTask(task,"MtCaptchaTask",proxy),"mtcaptcha")}async recaptcha(task,proxy){return await this.#executeTask(this.#convertRecaptchaTask(task,proxy),"recaptcha")}async turnstile(task,proxy){return await this.#executeTask(this.#convertTurnstileTask(task,proxy),"turnstile")}async text(task){return await this.#executeTask(this.#convertTextTask(task),"text")}async coordinate(task){return await this.#executeTask(this.#convertCoordinateTask(task),"coordinate")}async grid(task){return await this.#executeTask(this.#convertTask(task,"TextTask"),"text")}async slider(task){return await this.#executeTask(this.#convertTask(task,"TextTask"),"text")}async rotation(task){return await this.#executeTask(this.#convertTask(task,"TextTask"),"text")}};async function updateScraperConfig(config){const appCfg=appConfig.cfg,{baseDir:baseDir,inputFileDirPart:inputFileDirPart,useStealthPlugin:useStealthPlugin,browserControllerType:browserControllerType,browserType:browserType,lsdLaunchOptions:lsdLaunchOptions,lsdConnectOptions:lsdConnectOptions,browserConfigs:browserConfigs,urlPrefix:urlPrefix,maxConcurrency:maxConcurrency,readCode:readCode,templateParas:templateParas,totalMaxConcurrency:totalMaxConcurrency,minMiliseconds:minMiliseconds,exitWhenCompleted:exitWhenCompleted,useParasstrInXmlIfNeeded:useParasstrInXmlIfNeeded,loadUnfinishedTasks:loadUnfinishedTasks,loadFailedTasksInterval:loadFailedTasksInterval,moveDataWhenStart:moveDataWhenStart,dataFileFormat:dataFileFormat,columnSeperator:columnSeperator}=config;if(baseDir)appCfg.baseDir=baseDir;else{const cwd=process.cwd();appCfg.baseDir=path8.join(cwd,"scraperdata")}if(await(0,import_utils15.fsCheckOrCreateDir)(appCfg.baseDir,!0,!0),appCfg.baseDownloadDir=path8.join(appCfg.baseDir,"downloads"),await(0,import_utils15.fsCheckOrCreateDir)(appCfg.baseDownloadDir,!0,!0),appCfg.baseFileDir=path8.join(appCfg.baseDir,"files"),await(0,import_utils15.fsCheckOrCreateDir)(appCfg.baseFileDir,!0,!0),config.templateDir?appCfg.templateDir=config.templateDir:appCfg.templateDir=path8.join(appCfg.baseDir,"template"),await(0,import_utils15.fsCheckOrCreateDir)(appCfg.templateDir,!0,!0),appCfg.taskDir=path8.join(appCfg.baseDir,"task"),await(0,import_utils15.fsCheckOrCreateDir)(appCfg.taskDir,!0,!0),appCfg.exportsDir=path8.join(appCfg.baseDir,"exports"),await(0,import_utils15.fsCheckOrCreateDir)(appCfg.exportsDir,!0,!0),inputFileDirPart&&(appCfg.inputFileDirPart=inputFileDirPart),browserControllerType&&(appCfg.browserControllerType=browserControllerType),browserType&&(appCfg.browserType=browserType),lsdLaunchOptions&&"object"==typeof lsdLaunchOptions){appCfg.lsdLaunchOptions=Object.assign(appCfg.lsdLaunchOptions,lsdLaunchOptions);const addons=appCfg.lsdLaunchOptions.addons;Array.isArray(addons)&&addons.length>0&&(appCfg.lsdLaunchOptions.addons=addons.map(addon=>path8.isAbsolute(addon)?addon:path8.join(process.cwd(),addon)))}if(lsdConnectOptions&&"object"==typeof lsdConnectOptions&&(appCfg.lsdConnectOptions=Object.assign(appCfg.lsdConnectOptions,lsdConnectOptions)),browserConfigs&&(appCfg.browserConfigs=browserConfigs),urlPrefix&&(appCfg.urlPrefix=urlPrefix),maxConcurrency&&(appCfg.maxConcurrency=maxConcurrency),readCode&&(appCfg.readCode=readCode),templateParas&&(appCfg.templateParas=templateParas),totalMaxConcurrency&&(appCfg.totalMaxConcurrency=totalMaxConcurrency),"number"==typeof minMiliseconds&&(appCfg.minMiliseconds=minMiliseconds),"boolean"==typeof useStealthPlugin&&(appCfg.useStealthPlugin=useStealthPlugin),"boolean"==typeof exitWhenCompleted&&(appCfg.exitWhenCompleted=exitWhenCompleted),"boolean"==typeof useParasstrInXmlIfNeeded&&(appCfg.useParasstrInXmlIfNeeded=useParasstrInXmlIfNeeded),"boolean"==typeof loadUnfinishedTasks&&(appCfg.loadUnfinishedTasks=loadUnfinishedTasks),"number"==typeof loadFailedTasksInterval&&loadFailedTasksInterval>=0&&(appCfg.loadFailedTasksInterval=loadFailedTasksInterval),"boolean"==typeof moveDataWhenStart&&(appCfg.moveDataWhenStart=moveDataWhenStart),dataFileFormat&&(appCfg.dataFileFormat=dataFileFormat),"csv"===appCfg.dataFileFormat?appCfg.columnSeperator=",":"tsv"===appCfg.dataFileFormat?appCfg.columnSeperator="\t":"txt"===appCfg.dataFileFormat&&columnSeperator&&(appCfg.columnSeperator=columnSeperator),config.captcha){const{clientKey:clientKey,captchas:captchas}=config.captcha;"string"==typeof clientKey&&clientKey&&(appCfg.captcha.clientKey=clientKey),Array.isArray(captchas)&&captchas.every(name=>name&&"string"==typeof name)&&(appCfg.captcha.captchas=captchas)}return!0}async function scraper(newTasks=[],config={}){if(await updateScraperConfig(config),appConfig.cfg?.captcha?.clientKey){const twoCaptcha=new TwoCaptcha({clientKey:appConfig.cfg.captcha.clientKey});await addCapatchaProvider(twoCaptcha)}const browserManager=new BrowserManagerInScraper;await browserManager.init();const scheduler=new SchedulerInScraper(browserManager);return await scheduler.init(newTasks),!0}