@letsscrapedata/scraper 0.0.75 → 0.0.76

package/dist/index.cjs

@@ -1 +1 @@
-"use strict";var __create=Object.create,__defProp=Object.defineProperty,__getOwnPropDesc=Object.getOwnPropertyDescriptor,__getOwnPropNames=Object.getOwnPropertyNames,__getProtoOf=Object.getPrototypeOf,__hasOwnProp=Object.prototype.hasOwnProperty,__export=(t,e)=>{for(var r in e)__defProp(t,r,{get:e[r],enumerable:!0})},__copyProps=(t,e,r,a)=>{if(e&&"object"==typeof e||"function"==typeof e)for(let s of __getOwnPropNames(e))__hasOwnProp.call(t,s)||s===r||__defProp(t,s,{get:()=>e[s],enumerable:!(a=__getOwnPropDesc(e,s))||a.enumerable});return t},__toESM=(t,e,r)=>(r=null!=t?__create(__getProtoOf(t)):{},__copyProps(!e&&t&&t.__esModule?r:__defProp(r,"default",{value:t,enumerable:!0}),t)),__toCommonJS=t=>__copyProps(__defProp({},"__esModule",{value:!0}),t),src_exports={};__export(src_exports,{performOneTask:()=>performOneTask,scraper:()=>scraper,setScraperLogFun:()=>setScraperLogFun,updateScraperConfig:()=>updateScraperConfig}),module.exports=__toCommonJS(src_exports);var import_controller=require("@letsscrapedata/controller"),import_proxy=require("@letsscrapedata/proxy"),import_utils=require("@letsscrapedata/utils"),pkgLog=import_utils.log;function setScraperLogFun(t){return"function"==typeof t&&(pkgLog=t,(0,import_controller.setControllerLogFun)(pkgLog),(0,import_proxy.setProxyLogFun)(pkgLog),!0)}async function logdbg(...t){await pkgLog(import_utils.LogLevel.DBG,...t)}async function loginfo(...t){await pkgLog(import_utils.LogLevel.INF,...t)}async function logwarn(...t){await pkgLog(import_utils.LogLevel.WRN,...t)}async function logerr(...t){await pkgLog(import_utils.LogLevel.ERR,...t)}(0,import_controller.setControllerLogFun)(pkgLog),(0,import_proxy.setProxyLogFun)(pkgLog),(0,import_utils.setLogLevel)(import_utils.LogLevel.INF);var import_utils11=require("@letsscrapedata/utils"),import_xml2js_lsd=__toESM(require("xml2js-lsd"),1),myfunCfg={type:void 0,arg2:"",arg3:"",arg4:"",arg5:""},defaultSplitOfParases="[\r\n\t,]+",defaultSeperatorOfParas="::",newLineChar="\n",defaultElementCfg={template:{tid:void 0,version:void 0,commited:!1},attrs:{},attr:{name:void 0,value:void 0},paras:{},para:{paraname:void 0,name:"",desc:"",uitype:"",defval:"",listid:"0",listparas:"",min:"",max:"",pattern:"",alert:""},element:{loc:void 0,source:"default",absolute:!1},iframe:{srcprefix:"",loc:""},templstr:{templ:void 0},elecontent_attr:{type:void 0,attrname:void 0,attrname2:"",attrname3:"",attrname4:"",trim:!1,pattern:"",boolattr:!1,multieles:!0,join:newLineChar,line:!1,absolute:!1},elecontent_imgbase64:{type:void 0},elecontent_innerhtml:{type:void 0},elecontent_length:{type:void 0},elecontent_outerhtml:{type:void 0},elecontent_text:{type:void 0,trim:!0,multieles:!0,join:newLineChar,line:!1,top:!1},decodefontsvg:{outtype:"text"},decodefontttf:{fflocs:"",intype:"outerhtml",outtype:"text"},ocrcfg_api:{type:void 0,nlchars:"",seperator:!1,lang:"zh"},ocrcfg_tesseract:{type:void 0,nlchars:"",seperator:!1,lang:"zh"},actions:{},action_break:{type:void 0,id:"0"},action_captcha:{type:void 0,try:"1",errname:"captchafailed",id:"0"},action_click:{type:void 0,wait:"-1",gen:!0,cap:!0,clicktype:"click",try:"1",errname:"cfginvalid",datapage:"current",popupsubtask:!1,login:!0,captcha:!0,navigate:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",downloadpath:"",filename:"",pathvarname:"",id:"0"},action_continue:{type:void 0,id:"0"},action_misc:{type:void 0,id:"0"},misc_delsyncdbdata:{type:void 0,tids:"",retention:"2592000",minretention:"86400"},misc_extractdata:{type:void 0,name:void 0,method:"put",paras:!0,execdata:!0,subtasks:!0,credits:!0},misc_getstatedata:{type:void 0,cookies:"new",localstorage:"new",headers:"none",userdata:"none"},action_exit:{type:void 0,errname:void 0,id:"0"},action_extract:{type:void 0,tabname:void 0,id:"0"},column_element:{type:void 0,colname:void 0,nickname:"",completed:!1,setvar:!1,datatype:"string",errname:"ignore",valerrname:"ignore",pattern:"",flags:"",id:"0",discarded:!1},column_ocr:{type:void 0,colname:void 0,nickname:void 0,completed:!1,setvar:!1,datatype:"string",valerrname:"ignore",pattern:"",flags:"",id:"0",discarded:!1},column_subtask:{type:void 0,colname:void 0,nickname:void 0,completed:!1,setvar:!1,datatype:"string",valerrname:"ignore",pattern:"",flags:"",id:"0",discarded:!1},column_templstr:{type:void 0,colname:void 0,nickname:void 0,completed:!1,setvar:!1,datatype:"string",valerrname:"ignore",pattern:"",flags:"",id:"0",discarded:!1},action_goto:{type:void 0,url:void 0,reuse:!1,wait:"-1",encodeuri:!1,gen:!0,cap:!0,datapage:"current",popupsubtask:!1,login:!0,captcha:!0,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",headers:"",referer:"",id:"0"},action_hover:{type:void 0,try:"1",gen:!1,cap:!1,wait:"-1",errname:"cfginvalid",id:"0"},action_ifelse:{type:void 0,id:"0"},condition_element:{type:void 0},condition_else:{type:void 0},condition_templstr:{type:void 0},action_input:{type:void 0,content:void 0,enter:!1,replace:!1,gen:!1,cap:!1,try:"1",wait:"-1",errname:"cfginvalid",datapage:"current",popupsubtask:!1,login:!1,captcha:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",id:"0"},action_intercept_clear:{type:void 0,subtype:void 0},action_intercept_set:{type:void 0,subtype:void 0},request_abort:{type:void 0,method:"all",url:"",resourcetype:"image",postdata:""},response_cache:{type:void 0,method:"all",url:"",resourcetype:"xhr",postdata:"",minsize:"0",maxsize:"0",contenttype:"text",responses:!0,tabname:"",requestheaders:!1,responseheaders:!1},response_save:{type:void 0,method:"all",url:"",resourcetype:"image",postdata:"",minsize:"0",maxsize:"0",basedir:"response",pathtype:"path_name",hashmethod:"MD5",hostnameparts:"0",extname:"",validextnames:"",params:"",encoding:"",tabname:""},action_loopdowhile_element:{type:void 0,subtype:void 0,iswhile:!1,varname:"",maxloops:"100",click:!1,navigate:!0,gen:!0,cap:!0,errname:"cfginvalid",wait:"-1",id:"0"},action_loopdowhile_templstr:{type:void 0,subtype:void 0,iswhile:!1,varname:"",maxloops:"100",id:"0"},action_loopfor:{type:void 0,from:void 0,to:void 0,step:"1",roundtype:"ceil",varname:void 0,maxloops:"0",errname:"ignore",id:"0"},action_loopineles:{type:void 0,varname:"",maxloops:"0",start:"1",end:"0",step:"1",errname:"ignore",id:"0"},action_loopinstr:{type:void 0,list:void 0,split:defaultSplitOfParases,varname:void 0,maxloops:"0",trim:!0,errname:"ignore",id:"0"},action_scroll_by:{type:void 0,subtype:void 0,height:"100",unit:"windowpct",maxtimes:"1",interval:"800",gen:!1,cap:!1,datapage:"current",popupsubtask:!1,login:!1,captcha:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",id:"0"},action_scroll_intoview:{type:void 0,subtype:void 0,gen:!1,cap:!1,errname:"cfginvalid",datapage:"current",popupsubtask:!1,login:!1,captcha:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",id:"0"},action_scroll_to:{type:void 0,subtype:void 0,height:"100",unit:"bodypct",gen:!1,cap:!1,datapage:"current",popupsubtask:!1,login:!1,captcha:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",id:"0"},action_select:{type:void 0,selecttype:"value",selectval:"",gen:!1,cap:!1,try:"1",wait:"-1",errname:"cfginvalid",datapage:"current",popupsubtask:!1,login:!1,captcha:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",id:"0"},dbquery:{dbname:void 0,sql:void 0,sqlparas:"",parasplit:defaultSeperatorOfParas,join:newLineChar},action_setvar_dbquery:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},action_setvar_element:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",try:"1",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},file:{path:void 0,encoding:"utf8"},action_setvar_file:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",proxy:!0,valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},action_setvar_get:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},action_setvar_ocr:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},action_setvar_subtask:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},action_setvar_templstr:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},subtask:{tid:void 0,parasstr:void 0,idx:"",minlen:"1",errname:"cfginvalid",popuppage:!1},action_subtask:{type:void 0,subtasks:"",sameasparent:!1,id:"0"},action_wait_element:{type:void 0,subtype:void 0,timeout:"30000",state:"visible",errname:"ignore",wait:"-1",id:"0"},action_wait_navigation:{type:void 0,subtype:void 0,timeout:"30000",waituntil:"load",url:"",errname:"cfginvalid",wait:"-1",id:"0"},action_wait_sleep:{type:void 0,subtype:void 0,minms:"5000",maxms:"10000",errname:"ignore",wait:"-1",id:"0"},fonts:{},fontsvg:{exloc:"head link[rel='stylesheet']",inloc:"",csmaptype:"general",bsfilter:""},fontttf:{exloc:"head link[rel='stylesheet']",inloc:"",minuc:"57344",maxuc:"63744",startidx:"-1",fsfilter:"",fufilter:"",parsetype:""},fontselector:{name:void 0,fontfamily:void 0},fontfamily:{name:void 0,fontcodes:void 0,fontchars:void 0},fontcodes:{name:void 0,codes:void 0},fontchars:{name:void 0,chars:void 0},myfuns:{},myfun:{name:void 0,arg2:"",arg3:"",arg4:"",arg5:"",funbody:void 0},captcha_amazon:{type:void 0},captcha_funcaptcha:{type:void 0},captcha_geetest:{type:void 0},captcha_keycaptcha:{type:void 0},captcha_mtcaptcha:{type:void 0},captcha_recaptcha:{type:void 0,minscore:"0.3"},captcha_turnstile:{type:void 0},image_element:{type:void 0},comment_element:{type:void 0,attr:"textContent"},input_element:{type:void 0,enter:!1},submit_element:{type:void 0},check_result:{type:void 0,attr:"",failedstr:""},captcha_text:{type:void 0,case:!1,lang:"en",math:!1,maxlen:"0",minlen:"0",question:!1,space:!1,texttype:"any"},captcha_coordinate:{type:void 0,lang:"en",maxclicks:"0",minclicks:"1"},transform:{defaultval:"",usevar:!1},fun_c_attr:{type:void 0,subtype:void 0,loc:void 0,idx:"0",attrname:void 0},fun_c_data:{type:void 0,subtype:void 0,loc:void 0,idx:"0",attrname:void 0},fun_c_html:{type:void 0,subtype:void 0,loc:void 0,idx:"0"},fun_c_length:{type:void 0,subtype:void 0,loc:void 0},fun_c_text:{type:void 0,subtype:void 0,loc:void 0,idx:"-9999"},fun_ceil:{type:void 0},fun_closingsubstr:{type:void 0,openchar:void 0,startstr:"",position:"0"},fun_compress:{type:void 0,method:"gzip",flush:"",finishflush:"",chunksize:"0",windowbits:"0",level:"",memlevel:"",strategy:"",sourceencoding:"utf8",targetencoding:"base64"},fun_concat:{type:void 0,str1:void 0,str2:"",str3:"",str4:"",str5:""},fun_convertencoding:{type:void 0,sourceencoding:"base64",targetencoding:"utf8",start:"0",end:"0"},fun_decompress:{type:void 0,method:"gunzip",flush:"",finishflush:"",chunksize:"0",windowbits:"0",sourceencoding:"base64",targetencoding:"utf8",start:"0",end:"0"},fun_decode:{type:void 0,str1:"",val1:"",str2:"",val2:"",str3:"",val3:"",str4:"",val4:"",str5:"",val5:"",defaultval:""},fun_floor:{type:void 0},fun_hashcode:{type:void 0,hashmethod:"MD5"},fun_includes:{type:void 0,searchstr:void 0},fun_insert:{type:void 0,str:void 0},fun_itemstoobj:{type:void 0,split:defaultSplitOfParases,kvsplit:":",keytrim:!0,keynows:!1,valuetrim:!0,valuenows:!1},fun_jsonparse:{type:void 0,key1:"",key2:"",key3:"",key4:"",key5:"",key6:"",stringify:!1,keys:!1,length:!1,srctype:"json"},fun_length:{type:void 0},fun_matchall:{type:void 0,pattern:void 0,flags:"g",itemvalue:"match[0]",filter:"",start:"0",end:"0",resulttype:"all",join:newLineChar},fun_max:{type:void 0,split:defaultSplitOfParases},fun_min:{type:void 0,split:defaultSplitOfParases},fun_numeq:{type:void 0,num:void 0},fun_numge:{type:void 0,num:void 0},fun_numgt:{type:void 0,num:void 0},fun_numle:{type:void 0,num:void 0},fun_numlt:{type:void 0,num:void 0},fun_notincludes:{type:void 0,searchstr:void 0},fun_padend:{type:void 0,targetlen:void 0,padstr:" "},fun_padstart:{type:void 0,targetlen:void 0,padstr:" "},fun_parsefloat:{type:void 0},fun_parseint:{type:void 0},fun_randomarryidxes:{type:void 0,join:newLineChar},fun_replace:{type:void 0,substr:void 0,newstr:void 0,regexp:!1,flags:""},fun_round:{type:void 0},fun_setcomplement:{type:void 0,array2:void 0,split1:defaultSplitOfParases,split2:defaultSplitOfParases,sorttype:"nosort",valtype:"number",limit:"0",join:newLineChar},fun_setdifference:{type:void 0,array2:void 0,split1:defaultSplitOfParases,split2:defaultSplitOfParases,sorttype:"nosort",valtype:"number",limit:"0",join:newLineChar},fun_setintersection:{type:void 0,array2:void 0,split1:defaultSplitOfParases,split2:defaultSplitOfParases,sorttype:"nosort",valtype:"number",limit:"0",join:newLineChar},fun_setunion:{type:void 0,array2:void 0,split1:defaultSplitOfParases,split2:defaultSplitOfParases,sorttype:"nosort",valtype:"number",limit:"0",join:newLineChar},fun_slice:{type:void 0,beginidx:void 0,endidx:""},fun_shuffle:{type:void 0,split:defaultSplitOfParases},fun_sort:{type:void 0,split:defaultSplitOfParases,valtype:"number",sorttype:"asc",join:newLineChar},fun_specialconvert:{type:void 0,method:void 0,para1:"",para2:"",para3:"",para4:"",para5:""},fun_split:{type:void 0,split:defaultSplitOfParases,limit:"0",filter:"",start:"0",end:"0",resulttype:"all",val:"",insertstr:"",join:newLineChar},fun_streq:{type:void 0,str:void 0},fun_substrafter:{type:void 0,substr:void 0,position:"0",last:!1,include:!1},fun_substrbefore:{type:void 0,substr:void 0,position:"0",last:!1,include:!1},fun_tobool:{type:void 0},fun_tolowercase:{type:void 0},fun_tonum:{type:void 0},fun_touppercase:{type:void 0},fun_trim:{type:void 0},fun_trimend:{type:void 0},fun_trimstart:{type:void 0},fun_uniq:{type:void 0,split:defaultSplitOfParases,join:newLineChar},fun_urldelparams:{type:void 0,param1:void 0,param2:"",param3:""},fun_urldecode:{type:void 0,method:"URI",encoding:"utf8",percent:!0},fun_urlencode:{type:void 0,method:"URI",encoding:"utf8",percent:!0},fun_urlget:{type:void 0,attrorpath:void 0,hashmethod:"MD5",hostnameparts:"0",params:"",param:"",extname:"",validextnames:"",title:""},fun_urlsetparams:{type:void 0,param1:void 0,val1:"",param2:"",val2:"",param3:"",val3:"",replace:!0},fun_myfun1:Object.assign({},myfunCfg),fun_myfun2:Object.assign({},myfunCfg),fun_myfun3:Object.assign({},myfunCfg),fun_myfun4:Object.assign({},myfunCfg),fun_myfun5:Object.assign({},myfunCfg),get_addedtaskparas:{type:void 0,tid:void 0,limit:"10000",join:newLineChar,interval:"3600"},get_api:{type:void 0,method:"GET",url:void 0,encodeuri:!1,session:!1,headerssource:"headers",headers:"",referer:"",proxy:!0,timeout:"30000",context:"default",data:"",datatype:"json"},get_content:{type:void 0},get_cookies:{type:void 0,urls:"",domain:"",name:"",path:"",value:!1},get_datetime:{type:void 0,format:"yyyy-MM-ddThh:mm:ss"},get_execedtaskparas:{type:void 0,tid:void 0,limit:"10000",join:newLineChar,interval:"3600"},get_file:{type:void 0,url:void 0,path:"",basedir:"file",pathtype:"path_name",hashmethod:"MD5",hostnameparts:"0",params:"",extname:"",validextnames:"",pathvarname:"",proxy:!0,headers:"",referer:"",setvar:!1},httpbrowser:{name:void 0,minversion:"0",maxversion:"0",httpversion:"0"},httpdevice:{device:void 0},httpheader:{name:void 0,source:"value",value:void 0,delete:!1},httpos:{os:void 0},get_httpheaders:{type:void 0,headermethod:"genLatestWindowsChrome",origheaders:"",browserlist:"",httpversion:"0"},get_mhtml:{type:void 0,path:"",basedir:"mhtml",pathtype:"title",hashmethod:"MD5",hostnameparts:"0",pathvarname:""},get_pdf:{type:void 0,path:"",basedir:"pdf",pathtype:"title",hashmethod:"MD5",hostnameparts:"0",scale:"1",displayheaderfooter:!1,headertemplate:"",footertemplate:"",printbackground:!1,landscape:!1,pageranges:"",format:"Letter",width:"",height:"",top:"",right:"",bottom:"",left:"",screen:!1,onepage:!1,hmargin:"",setvar:!1,pathvarname:""},get_queuedcntwithparas:{type:void 0,tid:void 0,parasstr:void 0,operator:"eq"},get_queuedtaskparas:{type:void 0,tid:void 0,limit:"10000",join:newLineChar,interval:"3600"},get_random:{type:void 0,min:void 0,max:void 0},get_response:{type:void 0,requrl:"",idx:"-1",length:!1},get_screenshot:{type:void 0,basedir:"screenshot",path:"",extname:"png",quality:"-1",fullpage:!1,x:"0",y:"0",width:"0",height:"0",omitbackground:!1,setvar:!1,pathvarname:""},get_special:{type:void 0,method:void 0,parasstr:""},get_title:{type:void 0,aspath:!1},get_window:{type:void 0,key1:void 0,key2:"",key3:"",key4:"",key5:"",key6:"",keys:!1,length:!1}},appConfig={XML_ATTR_KEY:"$$",XML_CHILDREN_KEY:"children",GEN_CREDITS_BITS:16,GEN_CREDITS_AND:65535,CHECK_DNS_DOMAIN:"example.com",CHECK_POPUP_PAGE_TIMEOUT1:35e3,CHECK_POPUP_PAGE_TIMEOUT2:5e3,RESERVED_PARA_NAMES:["authInfo","inParas","sysParas"],EVENT_TYPES:{pagePopup:"pagePopup",pageClose:"pageClose",taskComplete:"taskComplete"},OPERATE_ACTIONS:["click","goto","hover","input","scroll","select","wait","captcha"],performOneTask:console.log,cfg:{exitWhenCompleted:!1,useParasstrInXmlIfNeeded:!1,loadUnfinishedTasks:!1,baseDir:"",baseDownloadDir:"",baseFileDir:"",templateDir:"",taskDir:"",exportsDir:"",inputFileDirPart:"LetsScrapeData",useStealthPlugin:!1,browserControllerType:"patchright",browserType:"chromium",lsdLaunchOptions:{headless:!1},lsdConnectOptions:{browserUrl:""},browserConfigs:[],captcha:{clientKey:""},maxConcurrency:1,readCode:"",templateParas:[],urlPrefix:"https://api.letsscrapedata.com/nologin",totalMaxConcurrency:10,minMiliseconds:2e3,moveDataWhenStart:!1,dataFileFormat:"jsonl",useNickName:!0,columnSeperator:"::"}},TaskParser=class t{static#t(e,r){try{if(!e||!r)throw new Error(`##te  Invalid paras, parentEleName ${r} in _applyDefaultElementCfg`);if("object"!=typeof e)return!0;const a=appConfig.XML_ATTR_KEY,s=appConfig.XML_CHILDREN_KEY;e[a]||(e[a]={});let i=r;e[a].type&&(i=i+"_"+e[a].type),e[a].subtype&&(i=i+"_"+e[a].subtype);const n=defaultElementCfg[i];if(!n)throw new Error(`##te  No default Cfg for eleName ${i} in _applyDefaultElementCfg`);const o=Object.assign({},n);if(Object.keys(n).forEach(t=>{let r=e[a][t];if("boolean"==typeof o[t])"true"===r||""===r?o[t]=!0:"false"===r&&(o[t]=!1);else if(r||""===r)o[t]=r;else if(void 0===o[t])throw new Error(`##te  Attribute ${t} of Element ${i} is undefined`)}),e[a]=o,!e[s])return!0;const c=Array.from(Object.keys(e[s]));for(const r of c){const a=e[s][r];1===a.length&&""===a[0]?e[s][r]=[]:a.forEach(e=>t.#t(e,r))}return!0}catch(t){throw logerr(`##te  Error parentEleName ${r} in _applyDefaultElementCfg`),new Error(t)}}static async convertXmlToJson(e,r=!0){try{const a=function(t){const e=t.indexOf("_");return e>0?t.slice(0,e):t},s={attrkey:appConfig.XML_ATTR_KEY,childkey:appConfig.XML_CHILDREN_KEY,explicitChildren:!0,tagNameProcessors:[a]},i=appConfig.XML_CHILDREN_KEY,n=await import_xml2js_lsd.default.parseStringPromise(e,s);if(!(n&&n.template&&n.template[i]&&Array.isArray(n.template[i].actions)))throw new Error(`##te  Invalid XML config string: ${e}`);return r&&t.#t(n.template,"template"),n.template}catch(t){throw logerr("##te  Exeception in convertXmlToJson"),logerr(`##te  xmlCfgStr: ${e}`),new Error(t)}}static getPartOfJsonCfg(t,e,r=!1){try{const r=t[appConfig.XML_CHILDREN_KEY][e][0];return r||null}catch(t){if(r)return null;throw logerr(`##te  Exeception when try to get ${e} in getPartOfJsonCfg`),new Error(t)}}static#e(e){try{const r=t.getPartOfJsonCfg(e,"attrs");return!Array.isArray(r?.children?.attr)||r.children.attr.length<=0?[]:r.children.attr.map(t=>t[appConfig.XML_ATTR_KEY])}catch(t){return logerr(t),[]}}static getParaCfgsFromJsonCfg(e){try{const r=t.getPartOfJsonCfg(e,"paras");return!Array.isArray(r?.children?.para)||r.children.para.length<=0?[]:r.children.para.map(t=>t[appConfig.XML_ATTR_KEY])}catch(t){return logerr(t),[]}}static getAttrsInXml(e){const r=t.#e(e),a={};return r.forEach(t=>a[t.name]=t.value),a}static#r(e,r){const a=[],s=appConfig.XML_CHILDREN_KEY;for(const i of e)if(i?.[appConfig.XML_ATTR_KEY]?.type===r)a.push(i);else if(i&&i[s]&&(i[s].action||i[s].condition)){t.#r(i[s].action?i[s].action:i[s].condition,r).forEach(t=>{a.push(t)})}return a}static getCaptchTypes(e){const r=appConfig.XML_CHILDREN_KEY,a=e[r].actions[0][r].action,s=t.#r(a,"captcha"),i=[];for(const t of s){const e=t[appConfig.XML_CHILDREN_KEY]?.captcha;Array.isArray(e)&&e.forEach(t=>{const e=t[appConfig.XML_ATTR_KEY].type;e&&i.push(e)})}return i}static getDatableMapFromJsonCfg(e){try{const r=appConfig.XML_CHILDREN_KEY,a=e[r].actions[0][r].action,s=t.#r(a,"extract"),i=new Map;for(const t of s){const e=t[appConfig.XML_ATTR_KEY].tabname;let a=i.get(e);a||(a=new Map,i.set(e,a));for(const e of t[r].column)if(!e[appConfig.XML_ATTR_KEY].discarded){const t=e[appConfig.XML_ATTR_KEY].colname,r=e[appConfig.XML_ATTR_KEY].nickname?e[appConfig.XML_ATTR_KEY].nickname:t;a.set(t,r)}}return i}catch(t){return null}}static getInParas(t,e=[],r="::"){const a={},s=t.split("@@"),i=["para","apara","rpara"];for(let t=0;t<3;t++){let n=i[t],o=e.filter(t=>t.paraname.startsWith(n)).length;if(s.length>t&&s[t].length>0){let e=s.length>t&&s[t].length>0?s[t].split(r):[];for(let t=1;t<=e.length;t++)a[`${n}${t}`]=e[t-1];for(let t=e.length+1;t<=o;t++)a[`${n}${t}`]=""}}return a}static convertExecData(t,e){if(!e)return t;const r={};for(const a of Object.keys(t)){const s=t[a],i=e.get(a);if(i&&0!==s.length){const t={};Object.keys(s[0]).forEach(e=>{const r=i.get(e);return t[e]=r||e});const e=s.map(e=>{const r={};for(const a of Object.keys(e))r[t[a]]=e[a];return r});r[a]=e}else r[a]=s}return r}},fs=__toESM(require("fs"),1),path=__toESM(require("path"),1),import_utils2=require("@letsscrapedata/utils"),TemplateManagerInScraper=class t{static#a=new Map;static#s=new Set;static#i(t){const e=appConfig.cfg.templateParas.find(e=>e.templateId===t);return e?.readCode?e.readCode:appConfig.cfg.readCode}static async#n(t,e){try{if(!e){if(!appConfig.cfg.templateDir)return null;const r=await(0,import_utils2.filesInDir)(appConfig.cfg.templateDir,"general",`templ${t}_`,"xml");let a=1===r.length?r[0]:`templ${t}.xml`;a=path.join(appConfig.cfg.templateDir,a),e=fs.readFileSync(a,"utf8")}if(!e)return null;return{templateId:t,domainId:1,defaultElementSource:"browser",maxExecutionDuration:600,configDetail:e}}catch(t){return null}}static async#o(e,r){try{if(t.#s.has(e))return null;let a=`${appConfig.cfg.urlPrefix}/template`;a=`${a}?tid=${e}&code=${r}`;const s=await fetch(a);if(200!==s.status)return 403===s.status?logwarn(`Failed to get template ${e}: No privilege`):logwarn(`Failed to get template ${e}, status: ${s.status}`),t.#s.add(e),null;const i=await s.json();if(!i||0!==i.code||!i.data)return logwarn(`Failed to get template ${e}: ${i.message}`),null;if(i.data.templateId!==e)return logerr(`Different templateId ${e} / ${i.templateId}`),null;const{domainId:n=0,defaultElementSource:o,configDetail:c="",maxExecutionDuration:l}=i.data;if(!c)return logerr(`Invalid content in template ${e}`),null;return{templateId:e,domainId:n,defaultElementSource:o,maxExecutionDuration:l,configDetail:c}}catch(t){return logdbg(t),null}}static async parseXmlTemplate(t,e=appConfig.cfg.useNickName){const r=await TaskParser.convertXmlToJson(t),a=TaskParser.getParaCfgsFromJsonCfg(r),s=TaskParser.getPartOfJsonCfg(r,"fonts",!0),i=TaskParser.getPartOfJsonCfg(r,"actions",!1),n=TaskParser.getAttrsInXml(r),o=TaskParser.getCaptchTypes(r),c=e?TaskParser.getDatableMapFromJsonCfg(r):null,l=(0,import_utils2.getCurrentUnixTime)();return{actionConfigs:i,paraCfgs:a,fontsConfig:s||null,attrsInXml:n,captchaTypes:o,datatableMap:c,lastUsedTime:l,lastCheckTime:l}}static async getTemplateConfig(e,r=""){if(!r){const r=t.#a.get(e);if(r)return r}let a=await t.#n(e,r);if(!a){const r=t.#i(e);a=await t.#o(e,r)}if(!a)throw new Error(`Cannot get template config of templateId ${e}`);const s=await t.parseXmlTemplate(a.configDetail),i=Object.assign({},s,{template:a});return t.#a.set(e,i),i}static clearTemplateConfig(e=0){return e>0?t.#a.delete(e):t.#a.clear(),!0}},import_node_events=__toESM(require("events"),1),import_node_events2=require("events"),fs4=__toESM(require("fs"),1),path4=__toESM(require("path"),1),import_utils9=require("@letsscrapedata/utils"),import_controller2=require("@letsscrapedata/controller"),uuid=__toESM(require("uuid"),1),SysParas=class{#c;#l;#p;#g;constructor(t,e,r,a){this.#c=t,this.#l=e,this.#p=r,this.#g=a}get version(){return this.#g}get templateId(){return this.#l}get taskId(){return this.#p}get url(){try{return this.#c?.page?this.#c.page.url():""}catch{return""}}get hasPopupPage(){return this.#c?.popupPage?"1":"0"}get subtaskNum(){const t=this.#c?.subtasks;return Array.isArray(t)?String(t.length):"0"}get datatableNum(){const t=this.#c?.execData;return t&&"object"==typeof t?String(Object.keys(t).length):"0"}get responses(){try{const t=this.#c?.responses;return Array.isArray(t)?JSON.stringify(t):""}catch(t){return""}}get currentDate(){return(new Date).toISOString().slice(0,10)}get currentDateTime(){return(new Date).toISOString().slice(0,19)}get currentIsoTime(){return(new Date).toISOString()}get currentLocalTime(){const t=new Date;return`${t.getFullYear()}-${String(t.getMonth()+1).padStart(2,"0")}-${String(t.getDate()).padStart(2,"0")} ${String(t.getHours()).padStart(2,"0")}:${String(t.getMinutes()).padStart(2,"0")}:${String(t.getSeconds()).padStart(2,"0")}`}get currentUtcTime(){return(new Date).toISOString().slice(0,19).replace("T"," ")}get currentTime(){return(new Date).toISOString().slice(11,19)}get currentUnixMs(){return String(Date.now())}get currentUnixSec(){return String(Math.floor(Date.now()/1e3))}get random10(){return String(Math.floor(10*Math.random()))}get random100(){return String(Math.floor(100*Math.random()))}get random1000(){return String(Math.floor(1e3*Math.random()))}get random10000(){return String(Math.floor(1e4*Math.random()))}get uuidv1(){return uuid.v1()}get uuidv4(){return uuid.v4()}},iconv=__toESM(require("iconv-lite"),1),zlib=__toESM(require("zlib"),1),cheerio=__toESM(require("cheerio"),1),import_utils3=require("@letsscrapedata/utils"),LsdString=class t{static specialConvertFunObj={};static setSpecialConvertFunObj(e){return t.specialConvertFunObj=e,!0}static transformStr(e,r,a,s){let i=e;try{if("string"!=typeof e||!Array.isArray(r))throw new Error("Invalid paras in transformStr");for(const e of r){if("object"!=typeof e)throw new Error("Invalid funCfg in transformStr");const r="c"===e.type?`${e.type}_${e.subtype}`:e.type;let a=t._funPerformers[r];!a&&e.type.match(/^myfun[0-9]$/)&&"object"==typeof s&&(a=s[e.type]),a&&"function"==typeof a?i=a(i,e):logerr(`Invalid funName ${e.type} in transformStr`)}}catch(t){loginfo(t)}return i||a}static __getElementBySelector(t,e,r=0){try{const a="root"===e?t.root():t(e);return a.length<=0||-9999===r?a:a.eq(r)}catch(t){return logerr("##te  cheerio  invalid $ or loc"),null}}static __cheerioOptions={xmlMode:!0,decodeEntities:!1};static _c_attr(e,r){try{const{loc:a,idx:s,attrname:i}=r,n=parseInt(s),o=cheerio.load(e,t.__cheerioOptions);return t.__getElementBySelector(o,a,n).attr(i)}catch(t){return logerr(`##tmp cheerio c_attr err: ${t}`),""}}static _c_data(e,r){try{const{loc:a,idx:s,attrname:i}=r,n=parseInt(s),o=cheerio.load(e,t.__cheerioOptions);return t.__getElementBySelector(o,a,n).data(i)}catch(t){return logerr(`##tmp cheerio c_data err: ${t}`),""}}static _c_html(e,r){try{const{loc:a,idx:s}=r,i=parseInt(s),n=cheerio.load(e,t.__cheerioOptions);return t.__getElementBySelector(n,a,i).html()}catch(t){return logerr(`##tmp cheerio c_html err: ${t}`),""}}static _c_length(e,r){try{const{loc:a}=r,s=-9999,i=cheerio.load(e,t.__cheerioOptions),n=t.__getElementBySelector(i,a,s);return String(n.length)}catch(t){return logerr(`##tmp cheerio c_length err: ${t}`),"0"}}static _c_text(e,r){try{const{loc:a,idx:s}=r,i=parseInt(s),n=cheerio.load(e,t.__cheerioOptions);return t.__getElementBySelector(n,a,i).text()}catch(t){return logerr(`##tmp cheerio c_text err: ${t}`),""}}static _ceil(t){return String(Math.ceil(Number(t)))}static _closingsubstr(t,e){const{openchar:r,startstr:a,position:s}=e;if("string"!=typeof t||"string"!=typeof r||!["{","[","(","lt"].includes(r)||"string"!=typeof a||"number"!=typeof parseInt(s)||parseInt(s)<0)return"";const i="lt"===r?"<":r,n={"{":"}","[":"]","(":")","<":">"}[i];let o=1,c=-1;if(a){if(c=t.indexOf(a),c<0)return""}else c=parseInt(s);const l=t.indexOf(i,c);if(l<0)return-1;let p=l+1;for(;p<t.length;p++){let e=t[p];if(e===i)o++;else if(e===n&&(o--,0===o))break}const g=p<=t.length?p+1:-1;return g>l?t.substring(l,g):""}static __getCompressOptions(t,e){return t||e?void 0:{}}static _compress(e,r){try{const{method:a,sourceencoding:s,targetencoding:i,start:n,end:o}=r,c=t.__getCompressOptions(r,!0),l=Buffer.from(e,s);let p=null;return"gzip"===a?p=zlib.gzipSync(l,c):"deflate"===a?p=zlib.deflateSync(l,c):"deflateraw"===a?p=zlib.deflateRawSync(l,c):"brotli"===a&&(p=zlib.brotliCompressSync(l,c)),p?t.__convertBufferToString(p,i,n,o):""}catch(t){return""}}static _decompress(e,r){try{const{method:a,sourceencoding:s,targetencoding:i,start:n,end:o}=r,c=t.__getCompressOptions(r,!1),l=Buffer.from(e,s);let p=Buffer.from(e,s);return"gunzip"===a?p=zlib.gunzipSync(l,c):"inflate"===a?p=zlib.inflateSync(l,c):"inflateraw"===a?p=zlib.inflateRawSync(l,c):"unzip"===a?p=zlib.unzipSync(l,c):"brotli"===a&&(p=zlib.brotliDecompressSync(l,c)),p?t.__convertBufferToString(p,i,n,o):""}catch(t){return""}}static _concat(t,e){let r=t;const{str1:a,str2:s,str3:i,str4:n,str5:o}=e;return[a,s,i,n,o].forEach(t=>{"string"==typeof t&&(r=r.concat(t))}),r}static _specialconvert(e,r){let a=e;try{const{method:s}=r;return"function"==typeof t.specialConvertFunObj[s]&&(a=t.specialConvertFunObj[s](e,r)),a}catch(t){return a}}static _decode(t,e){const{str1:r,val1:a,str2:s,val2:i,str3:n,val3:o,str4:c,val4:l,str5:p,val5:g,defaultval:h}=e;try{return t===r?a:t===s?i:t===n?o:t===c?l:t===p?g:h}catch(t){return""}}static __convertBufferToString(t,e,r,a){const s=r?Number(r):0,i=a?Number(a):0;return t instanceof Buffer?i>0?t.toString(e,s,i):s>0?t.toString(e,s):t.toString(e):""}static _convertencoding(e,r){const{sourceencoding:a,targetencoding:s,start:i,end:n}=r;try{const r=Buffer.from(e,a);return t.__convertBufferToString(r,s,i,n)}catch(t){return""}}static _floor(t){return String(Math.floor(Number(t)))}static _hashcode(t,e){const{hashmethod:r}=e;return import_utils3.LsdUrl.getHashCode(t,r)}static _includes(t,e){const{searchstr:r}=e;if("string"!=typeof r)throw new Error("Invalid searchstr in _contains");return String(Number(t.includes(r)))}static _insert(t,e){const{str:r}=e;if("string"!=typeof r)throw new Error("Invalid str in _insert");return r+t}static _itemstoobj(t,e){const{split:r,kvsplit:a,keytrim:s,keynows:i,valuetrim:n,valuenows:o}=e;if(!r||!a)return"{}";try{let e={};const c=t.split(new RegExp(r)),l=new RegExp(a);for(const t of c){const r=t.split(l);if(2!==r.length)continue;let a=s?r[0].trim():r[0];i&&(a=a.split("").filter(t=>t.trim()).join(""));let c=n?r[1].trim():r[1];o&&(c=c.split("").filter(t=>t.trim()).join("")),a&&(e[a]=c)}return JSON.stringify(e)}catch(t){return"{}"}}static _jsonparse(t,e){const{key1:r,key2:a,key3:s,key4:i,key5:n,key6:o,keys:c,length:l,srctype:p}=e;let g=null;try{if("script"===p)return"";if(g=JSON.parse(t),"object"!=typeof g||!g)return""}catch(t){return""}return g=(0,import_utils3.getMemberOfObject)(g,r,a,s,i,n,o),null==g?"":"string"==typeof g?g:"number"==typeof g?String(g):(c&&(g=Array.from(Object.keys(g))),l&&Array.isArray(g)?String(g.length):JSON.stringify(g))}static _length(t){return String(t.length)}static _matchall(t,e){try{const{pattern:r,flags:a,itemvalue:s,filter:i,start:n,end:o,resulttype:c,join:l}=e,p=new RegExp(r,a),g=t.matchAll(p);if("boolean"===c)return g?"1":"0";if(!g)return"";let h=[];for(const t of g)if("match[0]"===s)h.push(t[0]);else{let e=s;for(let r=0;r<t.length;r++)e=e.replaceAll(`match[${r}]`,t[r]);t.index>=0&&(e=e.replaceAll("match.index",String(t.index))),h.push(e)}if(i){const t=new RegExp(i);h=h.filter(e=>e.match(t))}const d=parseInt(n),f=parseInt(o);return 0===d&&0===f||(h=0===f?h.slice(d):h.slice(d,f)),"all"===c?h.join(l):"length"===c?String(h.length):""}catch(t){return loginfo(t),""}}static _max(t,e){try{const{split:r}=e,a=t.split(new RegExp(r)).filter(t=>t.trim()).map(t=>Number(t));return a.length>0?String(a.sort((t,e)=>t-e).pop()):""}catch(t){return loginfo(t),""}}static _min(t,e){try{const{split:r}=e,a=t.split(new RegExp(r)).filter(t=>t.trim()).map(t=>Number(t));return a.length>0?String(a.sort((t,e)=>t-e)[0]):""}catch(t){return loginfo(t),""}}static _numeq(t,e){const{num:r}=e;return"string"!=typeof t||"string"!=typeof r?"0":t.trim().match(/^\d+$/)&&r.trim().match(/^\d+$/)&&Number(t)===Number(r)?"1":"0"}static _numge(t,e){const{num:r}=e;return"string"!=typeof t||"string"!=typeof r?"0":t.trim().match(/^\d+$/)&&r.trim().match(/^\d+$/)&&Number(t)>=Number(r)?"1":"0"}static _numgt(t,e){const{num:r}=e;return"string"!=typeof t||"string"!=typeof r?"0":t.trim().match(/^\d+$/)&&r.trim().match(/^\d+$/)&&Number(t)>Number(r)?"1":"0"}static _numle(t,e){const{num:r}=e;return"string"!=typeof t||"string"!=typeof r?"0":t.trim().match(/^\d+$/)&&r.trim().match(/^\d+$/)&&Number(t)<=Number(r)?"1":"0"}static _numlt(t,e){const{num:r}=e;return"string"!=typeof t||"string"!=typeof r?"0":t.trim().match(/^\d+$/)&&r.trim().match(/^\d+$/)&&Number(t)<Number(r)?"1":"0"}static _notincludes(t,e){const{searchstr:r}=e;if("string"!=typeof r)throw new Error("Invalid str in _notcontains");return String(Number(!t.includes(r)))}static _padEnd(t,e){const{targetlen:r,padstr:a}=e,s=parseInt(r),i="string"==typeof a&&a.length>0?a:" ";return t.padEnd(s,i)}static _padStart(t,e){const{targetlen:r,padstr:a}=e,s=parseInt(r),i="string"==typeof a&&a.length>0?a:" ";return t.padStart(s,i)}static _parseFloat(t){return String(parseFloat(t))}static _parseInt(t){return String(parseInt(t))}static _randomarryidxes(t,e){const{join:r}=e;try{const e=parseInt(t);return[...Array(e).keys()].sort(()=>Math.random()-.5).join(r)}catch(t){return""}}static _replace(t,e){const{substr:r,newstr:a,regexp:s,flags:i}=e;try{const e=s;if("string"!=typeof r)throw new Error("Invalid substr in _replace");if("string"!=typeof a)throw new Error("Invalid newstr in _replace");return e?t.replace(RegExp(r,i),a):t.replace(r,a)}catch(e){return t}}static _round(t){return String(Math.round(Number(t)))}static __sort(t,e,r){let a;return a="string"===e?"asc"===r?t.sort((t,e)=>t.localeCompare(e)):t.sort((t,e)=>e.localeCompare(t)):"asc"===r?t.sort((t,e)=>parseFloat(t)-parseFloat(e)):t.sort((t,e)=>parseFloat(e)-parseFloat(t)),a}static _setcomplement(e,r){const{array2:a,split1:s,split2:i,sorttype:n,valtype:o,limit:c,join:l}=r;let p,g;try{p=new RegExp(s),g=new RegExp(i)}catch(t){throw new Error(`Invalid regexp split1: ${s}, split2: ${i} in _setcomplement`)}try{const r=parseInt(c);if(!["nosort","asc","desc"].includes(n)||!["string","number"].includes(o)||r<0||"string"!=typeof a||"string"!=typeof l)throw new Error(`Invalid limit: ${c}, sorttype: ${n}, or array2 in _setcomplement`);const s=e.split(p),i=a.split(g);let h=(0,import_utils3.getComplementOfTwoArray)(s,i);return["asc","desc"].includes(n)&&(h=t.__sort(h,o,n)),r>0?h.slice(0,r).join(l):h.join(l)}catch(t){return loginfo(t),e}}static _setdifference(e,r){const{array2:a,split1:s,split2:i,sorttype:n,valtype:o,limit:c,join:l}=r;let p,g;try{p=new RegExp(s),g=new RegExp(i)}catch(t){throw new Error(`Invalid regexp split1: ${s}, split2: ${i} in _setcomplement`)}try{const r=parseInt(c);if(!["nosort","asc","desc"].includes(n)||!["string","number"].includes(o)||r<0||"string"!=typeof a||"string"!=typeof l)throw new Error(`Invalid limit: ${c}, sorttype: ${n}, or array2 in _setcomplement`);const s=e.split(p),i=a.split(g);let h=(0,import_utils3.getDifferenceOfTwoArray)(s,i);return["asc","desc"].includes(n)&&(h=t.__sort(h,o,n)),r>0?h.slice(0,r).join(l):h.join(l)}catch(t){return loginfo(t),e}}static _setintersection(e,r){const{array2:a,split1:s,split2:i,sorttype:n,valtype:o,limit:c,join:l}=r;let p,g;try{p=new RegExp(s),g=new RegExp(i)}catch(t){throw new Error(`Invalid regexp split1: ${s}, split2: ${i} in _setcomplement`)}try{const r=parseInt(c);if(!["nosort","asc","desc"].includes(n)||!["string","number"].includes(o)||r<0||"string"!=typeof a||"string"!=typeof l)throw new Error(`Invalid limit: ${c}, sorttype: ${n}, or array2 in _setcomplement`);const s=e.split(p),i=a.split(g);let h=(0,import_utils3.getIntersectionOfTwoArray)(s,i);return["asc","desc"].includes(n)&&(h=t.__sort(h,o,n)),r>0?h.slice(0,r).join(l):h.join(l)}catch(t){return loginfo(t),e}}static _setunion(e,r){const{array2:a,split1:s,split2:i,sorttype:n,valtype:o,limit:c,join:l}=r;let p,g;try{p=new RegExp(s),g=new RegExp(i)}catch(t){throw new Error(`Invalid regexp split1: ${s}, split2: ${i} in _setcomplement`)}try{const r=parseInt(c);if(!["nosort","asc","desc"].includes(n)||!["string","number"].includes(o)||r<0||"string"!=typeof a||"string"!=typeof l)throw new Error(`Invalid limit: ${c}, sorttype: ${n}, or array2 in _setcomplement`);const s=e.split(p),i=a.split(g);let h=(0,import_utils3.getUnionOfTwoArray)(s,i);return["asc","desc"].includes(n)&&(h=t.__sort(h,o,n)),r>0?h.slice(0,r).join(l):h.join(l)}catch(t){return loginfo(t),e}}static _shuffle(t,e){try{const{split:r}=e;if("string"!=typeof r)throw new Error("Invalid split in _shuffle");const a=t.split(new RegExp(r));return!Array.isArray(a)||a.length<=1?t:a.sort(()=>Math.random()-.5).join(r)}catch(e){return loginfo(e),t}}static _slice(t,e){const{beginidx:r,endidx:a}=e,s=parseInt(r),i="string"==typeof a&&a.length>0?parseInt(a):void 0;return t.slice(s,i)}static _sort(t,e){try{const{split:r,valtype:a,sorttype:s,join:i}=e;if("string"!=typeof r||!["string","number"].includes(a)||!["asc","desc"].includes(s))throw new Error("Invalid split or valtype in _sort");const n=t.split(new RegExp(r));return!Array.isArray(n)||n.length<=1?t:"string"===a?"asc"===s?n.sort((t,e)=>t.localeCompare(e)).join(i):n.sort((t,e)=>e.localeCompare(t)).join(i):"asc"===s?n.sort((t,e)=>parseFloat(t)-parseFloat(e)).join(i):n.sort((t,e)=>parseFloat(e)-parseFloat(t)).join(i)}catch(t){return loginfo(t),""}}static _split(t,e){try{const{split:r,limit:a,filter:s,start:i,end:n,resulttype:o,val:c,insertstr:l,join:p}=e;if("string"!=typeof r)throw new Error("Invalid split in _split");const g=new RegExp(r),h=parseInt(a);let d=h>0?t.split(g,h):t.split(g);if(!Array.isArray(d))return"";if(s){const t=new RegExp(s);d=d.filter(e=>e.match(t))}const f=parseInt(i),u=parseInt(n);if(0===f&&0===u||(d=0===u?d.slice(f):d.slice(f,u)),l&&(d=d.map(t=>`${l}${t}`)),"all"===o)return d.join(p);if("length"===o)return String(d.length);if("index"===o){const t=d.findIndex(t=>t.trim()===c);return t>=0?String(t+1):""}return""}catch(t){return loginfo(t),""}}static _streq(t,e){const{str:r}=e;return"string"!=typeof r?"0":t===r?"1":"0"}static _substrAfter(t,e){const{substr:r,position:a,last:s,include:i}=e,n=i,o=parseInt(a);if("string"!=typeof r)throw new Error("Invalid substr in _substrAfter");let c=0;return c=s?o?t.lastIndexOf(r,o):t.lastIndexOf(r):o?t.indexOf(r,o):t.indexOf(r),-1===c?t:String(n?t.substring(c):t.substring(c+r.length))}static _substrBefore(t,e){const{substr:r,position:a,last:s,include:i}=e,n=parseInt(a),o=i;if("string"!=typeof r)throw new Error("Invalid substr in substrBefore");let c=0;return c=s?n?t.lastIndexOf(r,n):t.lastIndexOf(r):n?t.indexOf(r,n):t.indexOf(r),-1===c?t:o?t.substring(0,c+r.length):t.substring(0,c)}static _toBool(t){try{let e=t.replace(/true/g,"1");return e=e.replace(/false/g,"0"),e.search(/^[\s\d.+\-*/()<>=!|&:?]*$/)>-1&&-1===e.search(/=>/)?String(Number(Boolean((0,eval)(e)))):(loginfo(`Invalid logic expression ${t} in _toBool`),!1)}catch(t){return!1}}static _toLowerCase(t){return t.toLowerCase()}static _toNum(t){let e=t;return e.search(/^[\s\d.+\-*/()<>=!|&:?]+$/)>-1&&-1===e.search(/=>/)&&e.trim().length>0?String(Number((0,eval)(e))):(loginfo(`Invalid arithmetic expression ${e} in _toNum`),"0")}static _toUpperCase(t){return t.toUpperCase()}static _trim(t){return t.trim()}static _trimEnd(t){return t.trimEnd()}static _trimStart(t){return t.trimStart()}static _uniq(t,e){try{const{split:r,join:a}=e;if("string"!=typeof r||"string"!=typeof a)throw new Error("Invalid split in _uniq");const s=t.split(new RegExp(r));return!Array.isArray(s)||s.length<=1?t:[...new Set(s)].join(a)}catch(e){return loginfo(e),t}}static _urlencode(t,e){try{const{method:r,encoding:a,percent:s}=e;if("URI"===r)return encodeURI(t);if("URIComponent"===r)return encodeURIComponent(t);if("iconv"===r){const e=iconv.encode(t,a).toString("hex").toUpperCase();if(!e||e.length%2!=0)return logerr("Invalid str length in _urlencode"),"";if(s){const t=e.match(/../g);return t?t.map(t=>`%${t}`).join(""):""}return e}return logerr(`Invalid method ${r} in _urlencode`),""}catch(t){return logerr(t),""}}static _urldecode(t,e){try{const{method:r,encoding:a,percent:s}=e;if("URI"===r)return decodeURI(t);if("URIComponent"===r)return decodeURIComponent(t);if("iconv"===r){const e=s?/.../g:/../g,r=t.match(e);if(!r)return"";const i=r.map(t=>parseInt(t.slice(1),16));return iconv.decode(Buffer.from(i),a)}return logerr(`Invalid method ${r} in _urlencode`),""}catch(t){return logerr(t),""}}static _urldelparams(t,e){const{param1:r,param2:a,param3:s}=e;if("string"!=typeof r)throw new Error("Invalid params in _urldel");let i="";try{const e=new URL(t);if(!e||!e.searchParams||!e.searchParams.delete)return"";r&&e.searchParams.delete(r),a&&e.searchParams.delete(a),s&&e.searchParams.delete(s),i=e.href}catch(t){i=""}return i||""}static _urlget(t,e){const{attrorpath:r,hashmethod:a,param:s,hostnameparts:i,params:n,title:o,extname:c,validextnames:l}=e;if("string"!=typeof r)throw new Error("Invalid attr in _urlget");let p="";try{p=["href","origin","protocol","username","password","host","search","hash","hostname","port","hrefwithoutsearch","param"].includes(r)?import_utils3.LsdUrl.getUrlAttribute(t,r,i,s):import_utils3.LsdUrl.getFilePathFromUrl(t,r,a,i,n,c,l,o)}catch(t){p=""}return"string"==typeof p?p:""}static __setUrlParam(t,e,r,a){if(!t||!t.searchParams||!t.searchParams.get||"string"!=typeof e||"string"!=typeof r)return!1;if(!e)return!0;if(!a){if(null!==t.searchParams.get(e))return!0}return t.searchParams.set(e,r),!0}static _urlsetparams(e,r){const{param1:a,val1:s,param2:i,val2:n,param3:o,val3:c,replace:l}=r,p=l;if("string"!=typeof a)throw new Error("Invalid params in _urlset");let g="";try{const r=new URL(e);t.__setUrlParam(r,a,s,p),t.__setUrlParam(r,i,n,p),t.__setUrlParam(r,o,c,p),g=r.href}catch(t){g=""}return g||""}static _funPerformers={c_attr:t._c_attr,c_data:t._c_data,c_html:t._c_html,c_length:t._c_length,c_text:t._c_text,ceil:t._ceil,closingsubstr:t._closingsubstr,compress:t._compress,concat:t._concat,convertencoding:t._convertencoding,decode:t._decode,decompress:t._decompress,floor:t._floor,hashcode:t._hashcode,includes:t._includes,insert:t._insert,itemstoobj:t._itemstoobj,jsonparse:t._jsonparse,length:t._length,matchall:t._matchall,max:t._max,min:t._min,numeq:t._numeq,numge:t._numge,numgt:t._numgt,numle:t._numle,numlt:t._numlt,notincludes:t._notincludes,padend:t._padEnd,padstart:t._padStart,parsefloat:t._parseFloat,parseint:t._parseInt,randomarryidxes:t._randomarryidxes,replace:t._replace,round:t._round,setcomplement:t._setcomplement,setdifference:t._setdifference,setintersection:t._setintersection,setunion:t._setunion,shuffle:t._shuffle,slice:t._slice,sort:t._sort,specialconvert:t._specialconvert,split:t._split,streq:t._streq,substrafter:t._substrAfter,substrbefore:t._substrBefore,tobool:t._toBool,tolowercase:t._toLowerCase,tonum:t._toNum,touppercase:t._toUpperCase,trim:t._trim,trimend:t._trimEnd,trimstart:t._trimStart,uniq:t._uniq,urldecode:t._urldecode,urldelparams:t._urldelparams,urlencode:t._urlencode,urlget:t._urlget,urlsetparams:t._urlsetparams}};function getTransformExpStr(t,e,r="",a={}){return LsdString.transformStr(t,e,r,a)}var fs2=__toESM(require("fs"),1),path2=__toESM(require("path"),1),import_got_scraping=require("got-scraping"),import_header_generator=require("header-generator"),import_dateformat=__toESM(require("dateformat"),1),uuid2=__toESM(require("uuid"),1),import_utils4=require("@letsscrapedata/utils"),import_hpagent=require("hpagent"),download=__toESM(require("download"),1);async function lsdDownload(t,e="",r=null){const a={};if(e){const t=new import_hpagent.HttpsProxyAgent({keepAlive:!0,keepAliveMsecs:1e3,maxSockets:256,maxFreeSockets:256,scheduling:"lifo",proxy:e});a.agent={https:t}}let s;return r&&(a.headers=r),s=e||r?await download.default(t,void 0,a):await download.default(t),s}var GetPara=class t{static getSpecialFunObj={};static setGetSpecialFunObj(e){return t.getSpecialFunObj=e,!0}static getOtherFunObj={};static setGetOtherFunObj(e){return t.getOtherFunObj=e,!0}static noNeedToReturnContent="";static async getPara(e,r){if(!e||!e.type||!r)return"";const a=e.type;let s="";switch(a){case"addedtaskparas":"function"==typeof t.getOtherFunObj.getAddedTaskParasFun&&(s=await t.getOtherFunObj.getAddedTaskParasFun(e));break;case"api":s=await t.getApi(e,r);break;case"datetime":s=t.getDatetime(e);break;case"execedtaskparas":"function"==typeof t.getOtherFunObj.getExecedTaskParasFun&&(s=await t.getOtherFunObj.getExecedTaskParasFun(e));break;case"file":s=await t.getFile(e,r);break;case"httpheaders":s=await t.getHttpHeaders(e,r);break;case"queuedcntwithparas":"function"==typeof t.getOtherFunObj.getQueuedCntWithParasCfgFun&&(s=await t.getOtherFunObj.getQueuedCntWithParasCfgFun(e));break;case"queuedtaskparas":"function"==typeof t.getOtherFunObj.getQueuedTaskParasFun&&(s=await t.getOtherFunObj.getQueuedTaskParasFun(e));break;case"random":s=t.getRandom(e);break;case"response":s=await t.getResponse(e,r);break;case"special":s=await t.getSpecial(e,r);break;case"sysparas":s="please use ${sysParas.paraname}!!!";break;default:(0,import_utils4.unreachable)(a)}return s}static async getParaInElement(e,r,a){if(!e||!e.type||!a)return"";const s=e.type,i=a.paras;let n="";if("screenshot"===s)n=await t.getScreenshot(r,e,i);else(0,import_utils4.unreachable)(s);return n}static async getParaInPage(e,r,a){if(!e||!e.type||!a)return"";const s=e.type,i=a.paras;let n="";switch(s){case"content":n=await t.getContent(r);break;case"cookies":n="";break;case"mhtml":n=await t.getMhtml(r,e,i);break;case"pdf":n=await t.getPdf(r,e,i);break;case"screenshot":n=await t.getScreenshot(r,e,i);break;case"title":n=await t.getTitle(r,e);break;case"window":n=await t.getWindow(r,e);break;default:(0,import_utils4.unreachable)(s)}return n}static async getApi(e,r){const{method:a,url:s,encodeuri:i,session:n,headerssource:o,headers:c,referer:l,proxy:p,data:g,datatype:h}=e,d=e.context,f=e.timeout?Number(e.timeout):0;let u="";if(!["GET","DELETE","POST","PUT"].includes(a.toUpperCase()))return logerr(`##task  invalid method ${a} in getApi`),"";const m=s.startsWith("/");try{const e=i?encodeURI(s):s;if(m){if("function"==typeof t.getOtherFunObj.getDataFromServerFun){const r=f>0?{timeout:f}:void 0;return await t.getOtherFunObj.getDataFromServerFun(a,e,g,r)}return""}{const t={method:a.toUpperCase()};if(p){const e=r.proxyUrl?r.proxyUrl:"";e&&(t.proxyUrl=e)}n&&r.proxyUrl&&"default"===d&&(t.sessionToken=r.proxyUrl);const s="Content-Length";let i=!1,m={};if("headers"!==o||"headers"===o&&c||l){let a="";const n=r.interceptionData;if("headers"===o&&c?(m=JSON.parse(c),i="0"===m[s]):"cachedRequestHeaders"===o?(a=n.cachedRequestUrl,m=Object.assign({},n.cachedRequestHeaders),i=void 0!==m[s]):"cachedResponseHeaders"===o?(a=n.cachedResponseUrl,m=Object.assign({},n.cachedResponseHeaders),i=void 0!==m[s]):"origStateHeaders"===o?(a=n.origStateUrl,m=Object.assign({},n.origStateHeaders),i=void 0!==m[s]):"updatedRequestHeaders"===o?(a=n.updatedRequestUrl,m=Object.assign({},n.updatedRequestHeaders),i=void 0!==m[s]):"updatedResponseHeaders"===o?(a=n.updatedResponseUrl,m=Object.assign({},n.updatedResponseHeaders),i=void 0!==m[s]):"updatedStateHeaders"===o&&(a=n.updatedStateUrl,m=Object.assign({},n.updatedStateHeaders),i=void 0!==m[s]),o.endsWith("Headers")){const t=new URL(e),r=new URL(a);if(t.origin!==r.origin){if(!["origStateHeaders","updatedStateHeaders"].includes(o))return logerr(`Differenct url origins in getApi: req ${t.origin}, orig ${r.origin} `),"";if(!t.hostname.endsWith(r.hostname))return logerr(`Invalid url hostname in getApi: req ${t.hostname}, orig ${r.hostname} `),""}}l&&(m.referer=l),t.headers=m}if(g)if(i&&(m[s]=String(g.length)),"browser"===d||"state"===d||"task"===d||"standalone"===d)"json"===h?t.data=JSON.parse(g):"form"===h?t.form=JSON.parse(g):t.data=g;else if("default"===d)"json"===h?t.data=JSON.parse(g):"form"===h?t.form=JSON.parse(g):t.body=g;else if("fetch"===d)if("json"===h)t.data=JSON.parse(g);else if("form"===h){const e=new FormData,r=JSON.parse(g);for(const t of Object.keys(r))e.set(t,r[t]);t.data=e}else t.body=g;else logerr(`Invalid context ${d} in getApi`);if("browser"===d||"state"===d||"task"===d||"standalone"===d){const a=r.apiContext;if(!a)throw new Error("Null ApiContext in getApi");f>0&&(t.timeout=f);u=(await a.fetch(e,t)).text}else if("default"===d){f>0&&(t.timeout={request:f});u=(await(0,import_got_scraping.gotScraping)(e,t)).body}else if("fetch"===d){const r=await fetch(e,t);u=await r.text()}else logerr(`Invalid context ${d} in getApi`);return"string"==typeof u?u:JSON.stringify(u)}}catch(t){return loginfo(t),""}}static async getContent(t){try{if(!t)return"";return await t.content()}catch(t){return""}}static async getCookies(t,e){try{if(!t)return"";const{urls:r,domain:a,name:s,path:i,value:n}=e;let o="";r.length>0&&r.split(";");let c=await t.cookies();return a&&c.filter(t=>t.domain===a),s&&c.filter(t=>t.name===s),i&&c.filter(t=>t.path===i),o=n?c.length>0?c[0].value:"":JSON.stringify(c),o}catch(t){return""}}static getDatetime(t){const{format:e}=t;return(0,import_dateformat.default)(new Date,e)}static async getFile(e,r){const a=r.paras;try{const{url:s,pathtype:i,hashmethod:n,hostnameparts:o,params:c,extname:l,validextnames:p,basedir:g,setvar:h,pathvarname:d,proxy:f,headers:u,referer:m}=e;let y=e.path.trim();if(y)y=path2.join(y);else{let t=import_utils4.LsdUrl.getFilePathFromUrl(s,i,n,Number(o),c,l,p);y=path2.join(g,t)}if(y){const t=path2.dirname(y);if(!await(0,import_utils4.fsCheckOrCreateSubdir)(appConfig.cfg.baseFileDir,t))throw new Error("failed to checkOrCreateFileDir");y=path2.join(appConfig.cfg.baseFileDir,y)}const C=f&&r.proxyUrl?r.proxyUrl:"";let w=null;(u||m)&&(w=u?JSON.parse(u):{},m&&(w.referer=m));const k=await lsdDownload(s,C,w);return fs2.writeFileSync(y,k),d&&a&&!appConfig.RESERVED_PARA_NAMES.includes(d)&&(a[d]=y),h?k.toString("utf8"):t.noNeedToReturnContent}catch(t){throw new Error("error in get_file")}}static _genHeaderMethod={genLatestAndroid:import_header_generator.PRESETS.MODERN_ANDROID,genLatestDesktop:import_header_generator.PRESETS.MODERN_DESKTOP,genLatestLinux:import_header_generator.PRESETS.MODERN_LINUX,genLatestLinuxChrome:import_header_generator.PRESETS.MODERN_LINUX_CHROME,genLatestLinuxFirefox:import_header_generator.PRESETS.MODERN_LINUX_FIREFOX,genLatestMacos:import_header_generator.PRESETS.MODERN_MACOS,genLatestMacosChrome:import_header_generator.PRESETS.MODERN_MACOS_CHROME,genLatestMacosFirefox:import_header_generator.PRESETS.MODERN_MACOS_FIREFOX,genLatestMobile:import_header_generator.PRESETS.MODERN_MOBILE,genLatestWindows:import_header_generator.PRESETS.MODERN_WINDOWS,genLatestWindowsChrome:import_header_generator.PRESETS.MODERN_WINDOWS_CHROME,genLatestWindowsFirefox:import_header_generator.PRESETS.MODERN_WINDOWS_FIREFOX};static async getHttpHeaders(e,r){const{headermethod:a,origheaders:s}=e,{genHeaderOptions:i,newHttpHeaders:n}=r,o=r.interceptionData;let c={},l=!1;try{if(t._genHeaderMethod[a]){l=!0;c=new import_header_generator.HeaderGenerator(t._genHeaderMethod[a]).getHeaders()}else if("genNewByHttpOptions"===a){l=!0;c=new import_header_generator.HeaderGenerator(i).getHeaders()}else if("origheaders"===a)s&&(c=JSON.parse(s));else{if(!o)throw new Error("logic error: no interceptionData");"cachedRequestHeaders"===a?(c=Object.assign({},o.cachedRequestHeaders),o.updatedRequestUrl=o.cachedRequestUrl,o.updatedRequestHeaders=c):"cachedResponseHeaders"===a?(c=Object.assign({},o.cachedResponseHeaders),o.updatedResponseUrl=o.cachedResponseUrl,o.updatedResponseHeaders=c):"origStateHeaders"===a?(c=Object.assign({},o.origStateHeaders),o.updatedStateUrl=o.origStateUrl,o.updatedStateHeaders=c):"updatedRequestHeaders"===a?c=o.updatedRequestHeaders:"updatedResponseHeaders"===a?c=o.updatedResponseHeaders:"updatedStateHeaders"===a?c=o.updatedStateHeaders:logerr(`##get invalid httpMethod ${a} in getHttpHeaders`)}if(n)for(const t of Object.values(n)){const{name:e,source:r,value:s,deleteFlag:i}=t;if(i)delete c[e];else if("value"===r)c[e]=s;else if(!l&&r!==a){let t={};if("cachedRequestHeaders"===r)t=o.cachedRequestHeaders;else if("cachedResponseHeaders"===r)t=o.cachedResponseHeaders;else{if("origStateHeaders"!==r){logerr(`Invalid soure ${r} in httpheader`);continue}t=o.origStateHeaders}"string"==typeof t[e]?c[e]=t[e]:s||(c[e]=s)}}return l?JSON.stringify(c):""}catch(t){return logerr(t),""}}static async getMhtml(t,e,r){try{if(!t)return"";let a=e.path.trim();const{pathtype:s,hashmethod:i,hostnameparts:n,basedir:o,pathvarname:c}=e;if(a)a=path2.join(a);else{const e=t.url();let r=await t.title();a=import_utils4.LsdUrl.getFilePathFromUrl(e,s,i,Number(n),"","mhtml","",r),a=path2.join(o,a)}const{data:l}=await t.sendCDPMessage("Page.captureSnapshot",{format:"mhtml"});if(a){const t=path2.dirname(a);if(!await(0,import_utils4.fsCheckOrCreateSubdir)(appConfig.cfg.baseFileDir,t))throw new Error("failed to checkOrCreateMhtmlDir");const e=path2.join(appConfig.cfg.baseFileDir,a);fs2.writeFileSync(e,l),c&&r&&!appConfig.RESERVED_PARA_NAMES.includes(c)&&(r[c]=a)}return String(l)}catch(t){throw logdbg(t),new Error("error in get_mhtml")}}static _getScreenSize(t){return String(Number(t))===t?`${t}px`:t}static async getPdf(e,r,a){try{if(!e)return"";const{pathtype:s,hashmethod:i,hostnameparts:n,basedir:o,scale:c,displayheaderfooter:l,headertemplate:p,footertemplate:g,printbackground:h,landscape:d,pageranges:f,format:u,width:m,height:y,top:C,right:w,bottom:k,left:_,onepage:v,hmargin:S,pathvarname:b,setvar:x}=r;let E={},T=r.path.trim();if(T)T=path2.join(T);else{const t=e.url();let r=await e.title();T=import_utils4.LsdUrl.getFilePathFromUrl(t,s,i,Number(n),"","pdf","",r),T=path2.join(o,T)}if(T){const t=path2.dirname(T);if(!await(0,import_utils4.fsCheckOrCreateSubdir)(appConfig.cfg.baseFileDir,t))throw new Error("failed to checkOrCreatePdfDir");T=path2.join(appConfig.cfg.baseFileDir,T),E.path=T}if("1"!==c&&(E.scale=Number(c)),l&&(E.displayHeaderFooter=l),p&&(E.headerTemplate=p),g&&(E.footerTemplate=g),h&&(E.printBackground=h),d&&(E.landscape=d),f&&(E.pageRanges=f),C||w||k||_||l){const e={};C?e.top=t._getScreenSize(C):l&&!p&&(e.top="60px"),k?e.bottom=t._getScreenSize(k):l&&!g&&(e.bottom="60px"),w&&(e.right=t._getScreenSize(w)),_&&(e.left=t._getScreenSize(_)),E.margin=e}if(v){const t=await e.pageWidth();let r=await e.pageHeight();S&&(r+=Number(S)),E.width=`${t}px`,E.height=`${r}px`,logdbg(`--------- pageHeight：${r}px`)}else m||y?(m&&(E.width=t._getScreenSize(m)),y&&(E.height=t._getScreenSize(y))):"Letter"!==u&&(E.format=u);const A=await e.pdf(E);return b&&a&&!appConfig.RESERVED_PARA_NAMES.includes(b)&&(a[b]=T),x?A.toString("base64"):t.noNeedToReturnContent}catch(t){throw logdbg(t),new Error("error in get_pdf")}}static getRandom(t){try{const{min:e,max:r}=t,a=parseInt(e),s=parseInt(r);if(a>=s)return"0";const i=Math.round(Math.random()*(s-a))+a;return String(i)}catch(t){return"0"}}static async getResponse(t,e){try{const{requrl:r,idx:a,length:s}=t;let i=e?.interceptionData?.responses;if(!Array.isArray(i)||i.length<=0||!i.every(t=>t.requestUrl))return"";if(r){const t=new RegExp(r);i=i.filter(e=>e.requestUrl.match(t))}const n=i.length;if(n<=0)return"";if(s)return String(n);const o=parseInt(a);let c="";return o>=0&&o<n?c=i[o]?.responseData:o<0&&o+n>=0&&(c=i.slice(o)[0]?.responseData),"string"==typeof c?c:""}catch(t){throw new Error("error in get_response")}}static async getScreenshot(e,r,a){let s={};try{if(!e)return"";let{basedir:i,path:n,extname:o,quality:c,fullpage:l,x:p,y:g,width:h,height:d,omitbackground:f,pathvarname:u,setvar:m}=r;if(n=n.trim(),n)n=path2.join(i,n);else if(e.title){const t=await e.title(),r=`${import_utils4.LsdUrl.convertToValidPathname(t)}.${o}`;n=path2.join(i,r)}else{const t=`${uuid2.v4()}.${o}`;n=path2.join(i,t)}if(n){const t=path2.dirname(n);if(!await(0,import_utils4.fsCheckOrCreateSubdir)(appConfig.cfg.baseFileDir,t))throw new Error("failed to checkOrCreatePdfDir");n=path2.join(appConfig.cfg.baseFileDir,n),s.path=n}"jpeg"===o&&(s.type="jpeg");const y=parseInt(c);"jpeg"===o&&y>=0&&(s.quality=y),l&&(s.fullPage=l);const C=parseInt(h),w=parseInt(d);if(C>0&&w>0){const t={x:parseInt(p),y:parseInt(g),width:C,height:w};s.clip=t}f&&(s.omitBackground=f);const k=await e.screenshot(s);return u&&a&&!appConfig.RESERVED_PARA_NAMES.includes(u)&&(a[u]=n),m&&k?k.toString("base64"):t.noNeedToReturnContent}catch(t){throw new Error("error in get_screenshot")}}static async getSpecial(e,r){let a="";const s=e.method;return"function"==typeof t.getSpecialFunObj[s]&&(a=await t.getSpecialFunObj[s](e,r)),a}static async getTitle(t,e){try{if(!t)return"";const{aspath:r}=e,a=await t.title();return r?import_utils4.LsdUrl.convertToValidPathname(a):a}catch(t){return""}}static async getWindow(t,e){try{const{key1:r,key2:a,key3:s,key4:i,key5:n,key6:o,keys:c,length:l}=e;if(!t||!r)return"";const p=[r,a,s,i,n,o];let g=await t.windowMember(p);if(!c&&!l)return g;const h=JSON.parse(g);return h&&"object"==typeof h?c?Array.from(Object.keys(h)).join(","):l&&Array.isArray(h)?String(h.length):"":""}catch(t){return logwarn(t),""}}},path3=__toESM(require("path"),1),fsp=__toESM(require("fs/promises"),1),import_utils5=require("@letsscrapedata/utils");async function responseCacheCb(t,e,r){if(e&&t)try{const{execData:a,contentType:s,tabname:i,responseFlag:n,requestHeadersFlag:o,responseHeadersFlag:c,interceptionData:l}=e,{pageUrl:p,cookies:g}=r,h=t.request(),d=h.method(),f=h.url();let u=h.postData();const m=u||"",y="text"===s?await t.text():"";if(i&&(a[i]||(a[i]=[]),a[i].push({c01:p,c02:d,c03:f,c04:m,c05:y,c06:String(y.length)})),n&&l.responses.push({pageUrl:p,requestMethod:d,requestUrl:f,requestData:m,responseData:y}),o){const t=Object.assign({},h.allHeaders?await h.allHeaders():h.headers());!t.cookie&&g&&(t.cookie=g),l.cachedRequestUrl=f,l.cachedRequestHeaders=t,logdbg("requestUrl: ",f),logdbg("requestHeaders: ",t)}if(c){const e=Object.assign({},t.allHeaders?await t.allHeaders():t.headers());l.cachedResponseUrl=f,l.cachedResponseHeaders=e,logdbg("responseHeaders: ",e)}}catch(t){logerr(t)}}var isHttpResponse=t=>void 0!==t.buffer;async function responseSaveCb(t,e,r){if(!e||!t)return;const a=t.request(),s=a.url(),{execData:i,tabname:n,pathType:o,hashMethod:c,baseDir:l,hostnameParts:p,params:g,extname:h,validExtnames:d,encoding:f}=e,{pageUrl:u}=r;let m=import_utils5.LsdUrl.getFilePathFromUrl(s,o,c,p,g,h,d);s.startsWith("http")||logdbg("@@response: ",t),logdbg("@@reqUrl: ",s.slice(0,100)),logdbg("@@filename: ",m);let y=path3.join(l,m);if(!y)return;const C=path3.dirname(y);if(!await(0,import_utils5.fsCheckOrCreateSubdir)(appConfig.cfg.baseFileDir,C))throw new Error("failed to checkOrCreateReponseDir");const w=path3.join(appConfig.cfg.baseFileDir,y),k=isHttpResponse(t)?await t.buffer():await t.body();let _=f;if(!f){const t=a.resourceType();_=["document","stylesheet","script","xhr","fetch"].includes(t)?"utf8":"binary"}if(await fsp.writeFile(w,k,_),n&&i){const t=a.method();let e=a.postData();const r={c01:u,c02:t,c03:s,c04:e||"",c05:y,c06:String(k.length)};i[n]?i[n].push(r):i[n]=[r]}}var import_utils7=require("@letsscrapedata/utils"),import_events=__toESM(require("events"),1),import_utils6=require("@letsscrapedata/utils"),TwoCaptcha=class{#h;#d;#f;#u;#m;#y;async#C(t,e,r="POST"){try{const a={method:r,headers:{"Content-Type":"application/json"},body:JSON.stringify(e)},s=await fetch(t,a);if(!s.ok)return logerr(`##captcha failed to fetch, url ${t}`),null;const i=await s.json();return console.log("----------------------- fetch res: ",i),i}catch(t){return logerr(t),null}}async#w(t){const e={clientKey:this.#h,task:t,softId:0};"string"==typeof t.lang&&t.lang&&(e.languagePool=t.lang,delete t.lang),console.log("-----------------------  createTask body: ",e);return await this.#C("https://api.2captcha.com/createTask",e)}async#k(t){const e={clientKey:this.#h,taskId:t};return await this.#C("https://api.2captcha.com/getTaskResult",e)}async#_(){const t={clientKey:this.#h},e=await this.#C("https://api.2captcha.com/getBalance",t);return e?.balance>=0?e.balance:-1}#v(t){return`event${t}`}#S(t){return 5e3}async#b(t,e){const r={errorCode:"SUCCEEDED",taskId:0,cost:0,ip:"",createTime:(0,import_utils6.getCurrentUnixTime)(),endTime:0,solveCount:0},{errorId:a,taskId:s}=await this.#w(t);if(a>0||s<=0)return r.errorCode="ERROR_OTHER",s>0&&(r.taskId=s),r;r.taskId=s;const i=this.#v(s);await(0,import_utils6.sleep)(this.#S(e));let n=0;this.#u||(n=setInterval(async()=>{const t=await this.#k(s);console.log(`--------------------------- getResult of taskId ${s} @${new Date}: `,t),0!==t?.errorId?this.#y.emit(i,{code:1001,message:"failed",data:t}):0===t.errorId&&"ready"===t.status&&this.#y.emit(i,{code:0,message:"succeeded",data:t})},this.#d)),setTimeout(()=>this.#y.emit(i,{code:1009,message:"timeout",data:null}),this.#f);const[o]=await(0,import_events.once)(this.#y,i);console.log(`--------------------------- get eventData of taskId ${s} @${new Date}: `,o),n&&clearInterval(n),r.endTime=(0,import_utils6.getCurrentUnixTime)();const{code:c,data:l}=o;if(0===c){if(!l||0!==l.errorId)throw new Error("##captcha invalid data in getResult");const{cost:t,ip:e,createTime:a,endTime:s,solveCount:i,solution:n}=l;t&&(r.cost=Number(t)),e&&(r.ip=e),a&&a>0&&(a-r.createTime>5&&loginfo("##captcha task createTime difference: "+(a-r.createTime)),r.createTime=a),s&&s>0&&(r.endTime-s>2*this.#d&&loginfo("##captcha task endTime difference: "+(r.endTime-s)),r.endTime=s),i&&i>0&&(r.solveCount=i),n&&(r.solution=n)}else if(1001===c)r.errorCode=l?.errorCode?l.errorCode:"ERROR_OTHER";else{if(1009!==c)throw new Error(`##captcha unreachable at ${(new Error).stack}`);r.errorCode="ERROR_TIMEOUT"}return r}#x(t,e,r){let a=r?e:`${e}Proxyless`;return r?Object.assign({},t,{type:a},r):Object.assign({},t,{type:a})}#E(t,e){const{version:r,websiteURL:a,gt:s,challenge:i,geetestApiServerSubdomain:n,userAgent:o,captchaId:c}=t,l="GeeTestTask",p={type:e?l:`${l}Proxyless`,version:r,websiteURL:a};return s&&(p.gt=s),i&&(p.challenge=i),n&&(p.geetestApiServerSubdomainapiDomain=n),c&&(p.initParameters={captcha_id:c}),o&&(p.userAgent=o),p}#T(t,e){const{version:r,websiteKey:a,websiteURL:s,isEnterprise:i,isInvisible:n,s:o,action:c,minScore:l,userAgent:p,cookies:g,apiDomain:h}=t;let d="RecaptchaV3TaskProxyless";if(2===r){const t=i?"RecaptchaV2EnterpriseTask":"RecaptchaV2Task";d=e?t:`${t}Proxyless`}const f={type:d,websiteURL:s,websiteKey:a};return h&&(f.apiDomain=h),f.isInvisible=n,o&&(i?f.enterprisePayload=o:f.recaptchaDataSValue=o),p&&(f.userAgent=p),g&&(f.cookies=g),f.isEnterprise=i,l&&(f.minScore=l),c&&(f.pageAction=c),f}#A(t,e){const r="TurnstileTask";let a=e?r:`${r}Proxyless`;const{websiteKey:s,websiteURL:i,action:n,cData:o,chlPageData:c}=t;return n?{type:a,websiteKey:s,websiteURL:i,action:n,data:o,pagedata:c}:{type:a,websiteKey:s,websiteURL:i}}#I(t){const{images:e,comment:r,lang:a,math:s=!1,maxLength:i=0,minLength:n=0,question:o=!1,space:c=!1,textType:l="any"}=t;if(o)return{type:"TextCaptchaTask",comment:r,lang:a};{let o=0;"numbers"===l?o=1:"letters"===l&&(o=2);const p={type:"ImageToTextTask",body:e[0].toString("base64"),case:!!t.case,comment:r,math:s,maxLength:i,minLength:n,numeric:o,phrase:c,lang:a};return e.length>1&&(p.imgInstructions=e[1].toString("base64")),p}}#O(t){const{images:e,comment:r,lang:a,maxClicks:s=0,minClicks:i=1}=t,n={type:"CoordinatesTask",body:e[0].toString("base64"),comment:r,lang:a,minClicks:i};return s>0&&s>=i&&(n.maxClicks=s),e.length>1&&(n.imgInstructions=e[1].toString("base64")),n}constructor(t){const{clientKey:e,callbackUrl:r="",pollingInterval:a=5e3,timeout:s=15e4}=t;if(!e)throw new Error("Invalid options: no clientKey");this.#h=e,this.#d=a>=2e3?a:2e3,this.#f=s>=3e4?s:3e4,this.#u=r||"",r&&console.log(this.#u),this.#m=["amazon","funcaptcha","geetest","keycaptcha","mtcaptcha","recaptcha","turnstile","text","coordinate","grid","slider","rotation"],this.#y=new import_events.default}canSolveCaptcha(t){return this.#m.includes(t)}async getBalance(){return await this.#_()}name(){return"2captcha"}sendResult(t){if(!t||"object"!=typeof t)return!1;const{taskId:e,errorId:r}=t;if(e>0){const e=this.#v(t.taskId);return 0===r?this.#y.emit(e,{code:0,message:"succeeded",data:t}):0===r&&this.#y.emit(e,{code:1001,message:"failed",data:t}),!0}return!1}async amazon(t,e){return await this.#b(this.#x(t,"AmazonTask",e),"amazon")}async funcaptcha(t,e){return await this.#b(this.#x(t,"FunCaptchaTask",e),"funcaptcha")}async geetest(t,e){return await this.#b(this.#E(t,e),"geetest")}async keycaptcha(t,e){return await this.#b(this.#x(t,"KeyCaptchaTask",e),"keycaptcha")}async mtcaptcha(t,e){return await this.#b(this.#x(t,"MtCaptchaTask",e),"mtcaptcha")}async recaptcha(t,e){return await this.#b(this.#T(t,e),"recaptcha")}async turnstile(t,e){return await this.#b(this.#A(t,e),"turnstile")}async text(t){return await this.#b(this.#I(t),"text")}async coordinate(t){return await this.#b(this.#O(t),"coordinate")}async grid(t){return await this.#b(this.#x(t,"TextTask"),"text")}async slider(t){return await this.#b(this.#x(t,"TextTask"),"text")}async rotation(t){return await this.#b(this.#x(t,"TextTask"),"text")}},solvers=[];function getSolver(t){if(0===solvers.length){if(!appConfig.cfg?.captcha?.clientKey)throw new Error("Cannot solve captcha, please input clientKey of captcha provider.");{const t=new TwoCaptcha({clientKey:appConfig.cfg.captcha.clientKey});solvers.push(t)}}for(const e of solvers)if(e.canSolveCaptcha(t))return e;throw new Error(`##captcha  cannot solve ${t}`)}async function executeTask(t){const e=getSolver(t.type);let r;if(!e[t.type])throw new Error("##captcha logic error");return r=await e[t.type](t),console.log("---------------- response: ",r),r}var processors=new Map,captchaTypeAttr="data-captcha-type",widgetIdAttr="data-widget-id",captchaOptions={rootTag:"head",parentTag:"capwgts",captchaTag:"capwgt",solutionAttr:"value"};function checkCaptchaType(t,e){if(t.captchaType!==e)throw new Error(`##captcha invalid captchaType ${t.captchaType}`);return!0}function isValidStringAttr(t){return"string"==typeof t&&!!t&&!["null","undefined"].includes(t)}async function getCaptchasOnce(t,e,r=captchaOptions){let a=[];if(e){const r={opType:"get"},s=await t.evaluate(async([t,e])=>{const r=window[t];return"function"==typeof r?r(e):""},[e,r]);return Array.isArray(s)?(a=s,a):(logerr(`##captcha  no function window.${e}`),[])}{const e=`${r.rootTag} > ${r.parentTag} > ${r.captchaTag}`,s=await t.findElements(e);if(0===s.length)return a;for(const t of s){const e=await t.dataset();a.push(e)}return a}}async function getWidgets(t,e,r,a=captchaOptions){let s=await getCaptchasOnce(e,r,a);if(s.length>0)return s;const{needToRefresh:i,maxRefreshTimes:n}=t;if(i&&n)for(let t=0;t<n;t++){if(!await i(e))break;if(console.log(`---------------- page refreshed ${t+1} @${new Date}`),await e.reload(),await(0,import_utils7.sleep)(5e3),s=await getCaptchasOnce(e,r,a),s.length>0)return s}else if(await(0,import_utils7.sleep)(5e3),s=await getCaptchasOnce(e,r,a),s.length>0)return s;return[]}async function sendSolution(t,e,r,a,s=captchaOptions){const{captchaType:i,widgetId:n}=r;if(e){const r={opType:"execute",captchaType:i,widgetId:n,solution:a},s=await t.evaluate(async([t,e])=>{const r=window[t];return"function"==typeof r?r(e):""},[e,r]);return"string"==typeof s?s:(logerr(`##captcha  no function window.${e}`),"failed:noMsgFunc")}{const e=`${s.rootTag} > ${s.parentTag} > ${s.captchaTag}[${captchaTypeAttr}="${i}"][${widgetIdAttr}="${n}"]`,r=await t.findElement(e);if(r){return await r.setAttribute(s.solutionAttr,a)?"ok":"failed"}return"failed:noElement"}}var amazonGetTask=function(t){checkCaptchaType(t,"amazon");const{captchaType:e,pageurl:r,sitekey:a,iv:s,context:i,challengeScript:n,captchaScript:o}=t;return{type:e,websiteURL:r,websiteKey:a,iv:s,context:i,challengeScript:n,captchaScript:o}},amazonExecutionSolution=async function(t,e,r,a){const{existing_token:s}=a;return await sendSolution(t,e,r,s)},amazonProceesor={type:"amazon",findMethods:["intercept","hunt"],scriptName:"amazon",getTask:amazonGetTask,executeSolution:amazonExecutionSolution};processors.set("amazon",amazonProceesor);var geetestGetTask=function(t){checkCaptchaType(t,"geetest");const{captchaType:e,version:r,url:a,gt:s,challenge:i,apiServer:n,captchaId:o}=t;if(!r||!["v3","v4"].includes(r))throw new Error(`## captcha invalid version ${t.version} in ${t.captchaType}`);if(!a)throw new Error(`## captcha invalid url in ${t.captchaType}`);if(!("v3"!==r||s&&i))throw new Error(`## captcha invalid gt or challenge in ${t.captchaType} v3`);if("v4"===r&&!o)throw new Error(`## captcha invalid captchaId in ${t.captchaType} v4`);const c={type:e,version:"v4"===r?4:3,websiteURL:a};return isValidStringAttr(s)&&(c.gt=s),isValidStringAttr(i)&&(c.challenge=i),isValidStringAttr(n)&&(c.geetestApiServerSubdomain=n),isValidStringAttr(o)&&(c.captchaId=o),c},geetestExecutionSolution=async function(t,e,r,a){checkCaptchaType(r,"geetest");let s="";if("v3"===r.version){const{challenge:t,validate:e,seccode:i}=a,n=i||`${e}|jordan`;if("string"!=typeof t||!t)throw new Error(`##captcha invalid challenge in ${r.captchaType}`);if("string"!=typeof e||!e)throw new Error(`##captcha invalid validate in ${r.captchaType}`);if("string"!=typeof i||!i)throw new Error(`##captcha invalid seccode in ${r.captchaType}`);s=(0,import_utils7.encodeToBase64)({geetest_challenge:t,geetest_validate:e,geetest_seccode:n})}else{if("v4"!==r.version)return"failed:invalidVersion";{const{captcha_id:t,lot_number:e,pass_token:i,gen_time:n,captcha_output:o}=a;if("string"!=typeof t||!t)throw new Error(`##captcha invalid captcha_id in ${r.captchaType}`);if("string"!=typeof e||!e)throw new Error(`##captcha invalid lot_number in ${r.captchaType}`);if("string"!=typeof i||!i)throw new Error(`##captcha invalid pass_token in ${r.captchaType}`);if("string"!=typeof n||!n)throw new Error(`##captcha invalid gen_time in ${r.captchaType}`);if("string"!=typeof o||!o)throw new Error(`##captcha invalid captcha_output in ${r.captchaType}`);s=(0,import_utils7.encodeToBase64)({captcha_id:t,lot_number:e,pass_token:i,gen_time:n,captcha_output:o})}}return await sendSolution(t,e,r,s)},geetestProceesor={type:"geetest",findMethods:["intercept"],scriptName:"geetest",getTask:geetestGetTask,executeSolution:geetestExecutionSolution};processors.set("geetest",geetestProceesor);var recaptchaGetTask=function(t,e){checkCaptchaType(t,"recaptcha");const{captchaType:r,version:a,url:s,sitekey:i,action:n,s:o,enterprise:c,invisible:l,userAgent:p,cookies:g,apiDomain:h}=t;if(!a||!["v2","v3"].includes(a))throw new Error(`## captcha invalid version ${t.version} in ${t.captchaType}`);if(!s||!i)throw new Error(`## captcha invalid url or sitekey in ${t.captchaType}`);let d=.3;"number"==typeof e?.minScore&&e.minScore>0&&e.minScore<1&&(d=e.minScore);const f={type:r,version:"v3"===a?3:2,websiteURL:s,websiteKey:i,isEnterprise:"true"===c,isInvisible:"true"===l,minScore:d};return isValidStringAttr(n)&&(f.action=n),isValidStringAttr(o)&&(f.s=o),isValidStringAttr(p)&&(f.userAgent=p),isValidStringAttr(g)&&(f.cookies=g),isValidStringAttr(h)&&(f.apiDomain=h),f},recaptchaExecutionSolution=async function(t,e,r,a){checkCaptchaType(r,"recaptcha");const{gRecaptchaResponse:s}=a;if("string"!=typeof s||!s)throw new Error(`##captcha invalid gRecaptchaResponse in ${r.captchaType}`);return await sendSolution(t,e,r,s)},recaptchaProceesor={type:"recaptcha",findMethods:["intercept","hunt"],scriptName:"recaptcha",getTask:recaptchaGetTask,executeSolution:recaptchaExecutionSolution};processors.set("recaptcha",recaptchaProceesor);var turnstileNeedToRefresh=async function(t){return 0===(await t.findElements("#cf-turnstile")).length},turnstileGetTask=function(t){checkCaptchaType(t,"turnstile");const{captchaType:e,url:r,sitekey:a,action:s,cData:i,chlPageData:n}=t,o={type:e,websiteURL:r,websiteKey:a};return isValidStringAttr(s)&&(o.action=s),isValidStringAttr(i)&&(o.cData=i),isValidStringAttr(n)&&(o.chlPageData=n),o},turnstileExecutionSolution=async function(t,e,r,a){checkCaptchaType(r,"turnstile");const{token:s}=a;if("string"!=typeof s||!s)throw new Error(`##captcha invalid token in ${r.captchaType}`);return await sendSolution(t,e,r,s)},turnstileProceesor={type:"turnstile",findMethods:["intercept","hunt"],scriptName:"turnstile",getTask:turnstileGetTask,executeSolution:turnstileExecutionSolution,needToRefresh:turnstileNeedToRefresh,maxRefreshTimes:2};async function getScripFromServer(t,e,r){try{let e=`${appConfig.cfg.urlPrefix}/script?name=${t}`;const r=await fetch(e);if(200!==r.status)return"";const a=await r.json();if(!a||0!==a.code||!a.data||"string"!=typeof a.data)return"";return Buffer.from(a.data,"base64").toString("utf8")}catch(t){return loginfo(t),""}}processors.set("turnstile",turnstileProceesor);var scriptMap=new Map;async function getScript(t){let e=scriptMap.get(t);if(e)return e;if(e=await getScripFromServer(t),"string"==typeof e&&e.length>20)return scriptMap.set(t,e),e;throw new Error(`##captcha failed to get script of ${t}`)}async function injectScripts(t,e){let r=!0;if(!Array.isArray(e)||0===e.length)return r=!1,r;const a=[];for(const s of e){const e=processors.get(s);if(!e)continue;if(!e.findMethods.includes("intercept")||!e.scriptName)continue;if(a.includes(e.scriptName))continue;const i=await getScript(e.scriptName);if(!i){r=!1;break}a.push(e.scriptName);await t.addPreloadScript(i)||(r=!1)}return t.setCloseWhenFree(!0),r}async function solveTokenCaptcha(t,e,r,a){const s={code:0,captchaType:e,taskResults:[]};console.log(`######################### start to sovle ${e} @${new Date}: `,a);let i="ok";try{const r=processors.get(e);if(!r)return logerr(`##captcha cannot process ${e}`),s.code=1001,s;const{getTask:n,executeSolution:o}=r,c=await getWidgets(r,t,"");if(console.log("--------------- captchaWidgets: ",c),0===c.length)return s;const l=c[0];e!==l.captchaType&&logerr(`##captcha different captchaType ${e} ${l.captchaType}`);const p=n(l,a);console.log("--------------- task: ",p);const{solution:g,errorCode:h,taskId:d,createTime:f,endTime:u}=await executeTask(p);if(s.taskResults.push({errorCode:h,taskId:d,elapse:u-f}),g&&(i=await o(t,"",l,g),i.startsWith("ok")||(s.code=1003,s.message=i),i)){const e="button[type='submit']",r=await t.findElement(e);r&&console.log("@@@@@@@@@@@@@@@ click: ",await r.click())}return console.log(`######################### end to sovle ${e} @${new Date}: `,s),s}catch(t){return logerr(t),s.code=1010,s.message=t.message,console.log(`######################### end to sovle ${e} @${new Date}: `,s),s}}var import_node_fs=__toESM(require("fs"),1),import_utils8=require("@letsscrapedata/utils"),processors2=new Map;async function getTextOfElement(t,e,r){if("pageUrl"===r)return t.url();const a=await t.findElement(e);return a?"textContent"===r?await a.textContent():await a.attribute(r):""}async function getBufferOfImage(t,e,r,a){const s=await t.findElement(e);if(!s)return null;try{const e=await s.attribute("src");if(e.toLowerCase().includes(".gif")){const s=t.url(),i=import_utils8.LsdUrl.getAbsoulteUrl(s,e),n=await lsdDownload(i,r?.proxyUrl);return"number"==typeof a&&import_node_fs.default.writeFileSync(`./captchaImage${a}.gif`,n),n}{"lazy"===await s.attribute("loading")&&(await s.scrollIntoView(),await(0,import_utils8.sleep)(2e3));const t=await s.screenshot();return"number"==typeof a&&import_node_fs.default.writeFileSync(`./captchaImage${a}.png`,t),t}}catch(t){return logerr(t),null}}var textGetTask=async function(t,e,r=null){const{type:a,imageLocs:s,case:i,commentLoc:n,commentAttr:o,inputLoc:c,enter:l,submitLoc:p,checkLoc:g,lang:h,math:d,maxLength:f,minLength:u,question:m,space:y,textType:C}=e;if(!Array.isArray(s)||0===s.length)throw new Error("invalidParas");if(!l&&!p)throw new Error("invalidParas");if(!(!m||n&&o))throw new Error("invalidParas");if(!g)throw new Error("invalidParas");const w=[];for(const[e,a]of s.entries()){const s=await getBufferOfImage(t,a,r,e);if(!s)throw new Error("noRequiredElement:imageLoc");w.push(s)}if(!await t.findElement(c))throw new Error("noRequiredElement:inputLoc");let k="";if(n&&(k=await getTextOfElement(t,n,o),!k))throw new Error("noRequiredElement:commentLoc");if(p){if(!await t.findElement(p))throw new Error("noRequiredElement:submitLoc")}return{type:a,images:w,case:i,comment:k,lang:h,math:d,maxLength:f,minLength:u,question:m,space:y,textType:C,websiteUrl:t.url()}},textExecuteSolution=async function(t,e,r){const{text:a}=r,{inputLoc:s,enter:i,submitLoc:n,checkLoc:o,checkAttr:c,failedStr:l}=e,p=await t.findElement(s);if(!p)throw new Error("noRequiredElement");let g=!1;if(i){if(g=await p.input(a,{enter:i}),!g)return"failed:input"}else{const e=await t.findElement(n);if(!e)throw new Error("noRequiredElement");if(g=await p.input(a),!g)return"failed:input";if(g=await e.click(),!g)return"failed:submit"}await(0,import_utils8.sleep)(1e3);const h=await t.findElement(o);if(!h)return"ok";if(!c)return"failed:checkFailed-noCheckElement";const d="textContent"===c?await h.textContent():await h.attribute(c);return!l&&l.length>0?`failed:checkFailed-${d}`:d.includes(l)?`failed:checkFailed-${l}`:"ok"},textProcessor={type:"text",getTask:textGetTask,executeSolution:textExecuteSolution};processors2.set("text",textProcessor);var coordinateGetTask=async function(t,e,r=null){const{type:a,imageLocs:s,commentLoc:i,commentAttr:n,lang:o,maxClicks:c,minClicks:l}=e;if(!Array.isArray(s)||0===s.length)throw new Error("invalidParas");if(l<0||c>0&&l>=c)throw new Error("invalidParas");const p=[];for(const[e,a]of s.entries()){const s=await getBufferOfImage(t,a,r,e);if(!s)throw new Error("noRequiredElement:imageLoc");p.push(s)}let g="";if(i&&(g=await getTextOfElement(t,i,n),!g))throw new Error("noRequiredElement:commentLoc");return{type:a,images:p,comment:g,lang:o,maxClicks:c,minClicks:l}},coordinateExecuteSolution=async function(t,e,r){const{coordinates:a}=r,{minClicks:s,maxClicks:i,imageLocs:n,submitLoc:o,checkLoc:c,checkAttr:l,failedStr:p}=e;if(!Array.isArray(a)||a.length<s||i>0&&a.length>i)return"failed:invalidSolution";let g=!1;const h=await t.findElement(n[0]);if(!h)throw new Error("noRequiredElement");const d=await h.boundingBox();if(!d)return"failed:click";const{x:f,y:u}=d;for(const{x:e,y:r}of a){if(g=await t.mouseClick(f+e,u+r),!g)return"failed:click";await(0,import_utils8.sleep)(200)}if(await(0,import_utils8.sleep)(1e3),o){const e=await t.findElement(o);if(!e)throw new Error("noRequiredElement");if(g=await e.click(),!g)return"failed:submit";await(0,import_utils8.sleep)(1e3)}const m=await t.findElement(c);if(!m)return"ok";if(!l)return"failed:checkFailed-noCheckElement";const y="textContent"===l?await m.textContent():await m.attribute(l);return!p&&p.length>0?`failed:checkFailed-${y}`:y.includes(p)?`failed:checkFailed-${p}`:"ok"},coordinateProcessor={type:"text",getTask:coordinateGetTask,executeSolution:coordinateExecuteSolution};async function solveRecognitionCaptcha(t,e,r,a=1){const s=e.type;console.log(`######################### start to sovle ${s} @${new Date}: `,e);const i={code:0,captchaType:s,taskResults:[]};let n="";try{const o=processors2.get(s);if(!o)return logerr(`##captcha cannot process ${s}`),i.code=1001,i;const{getTask:c,executeSolution:l}=o;for(let s=1;s<=a;s++){const a=await c(t,e,r);console.log("--------------- task: ",a);const{solution:s,errorCode:o,taskId:p,createTime:g,endTime:h}=await executeTask(a),d={errorCode:o,taskId:p,elapse:h-g};if(i.taskResults.push(d),s){if(n=await l(t,e,s),n.startsWith("ok"))return i;if(n.startsWith("failed:checkFailed")){d.errorCode="ERROR_CHECK_FAILED";continue}d.errorCode="ERROR_SOLUTION_FAILED";break}}return i.code=1003,i.message=n,console.log(`######################### end to sovle ${s} @${new Date}: `,i),i}catch(t){logerr(t);const e=t.message;return"invalidParas"===e?i.code=1001:e.startsWith("noRequiredElement")?i.code=1004:i.code=1010,i.message=e,console.log(`######################### end to sovle ${s} @${new Date}: `,i),i}}function calcCapCredits(t){return t>0?t>>appConfig.GEN_CREDITS_BITS:0}function calcGenCredits(t){return t>0?t&appConfig.GEN_CREDITS_AND:0}processors2.set("coordinate",coordinateProcessor);var TaskScraper=class extends import_node_events.default{#D;#P;#R;#g;#l;#p;#$;#N;#L;#F;#M;#j;#c;_taskContext;#q;_paras;#U;#H;#B;#V;#W;constructor(t,e,r,a={}){super();const{template:s,actionConfigs:i,fontsConfig:n}=t,{proxy:o=null,page:c=null,browserApiContext:l=null,standaloneApiContext:p=null}=r,{performOneTask:g=appConfig.performOneTask,taskId:h=0,skipActions:d=[],extractColumns:f=[],version:u="1.0",splitStr:m="::",authInfo:y={}}=a,C=a.stateData?a.stateData:{cookies:[],localStorage:[],headers:{},userData:{}},w=a.defaultElementSource?a.defaultElementSource:s.defaultElementSource;try{this.#D=s,this.#P=i,this.#R=n,this.#g=u,this.#l=s.templateId,this.#p=h,this.#$=Array.isArray(a._captchaOptions?.captchaTypes)?a._captchaOptions.captchaTypes:[],this.#N=w,this.#F=d,this.#M=f,this.#L=m,this.#j=g,this.#c={proxy:o,page:c,browserApiContext:l,standaloneApiContext:p,stateApiContext:l||p,taskApiContext:null,popupPage:null,browserElesStack:[],cheerioPage:null,cheerioElesStack:[],loopsStack:[],continueFlag:!1,breakFlag:!1,origStateData:C,newStateData:null,execData:{},subtasks:[],credits:0,subtaskResults:[]};const t=new SysParas(this.#c,this.#l,this.#p,this.#g);this.#q={inParas:e,sysParas:t,authInfo:y,userData:Object.assign({},C.userData),responses:[]},this.#U=this.#K();const r=C.localStorage.length>0?C.localStorage[0].origin:C.cookies.length>0?`https://${C.cookies[0].domain}`:"";this.#H={responses:[],cachedRequestUrl:"",cachedRequestHeaders:{},cachedResponseUrl:"",cachedResponseHeaders:{},updatedRequestUrl:"",updatedRequestHeaders:{},updatedResponseUrl:"",updatedResponseHeaders:{},origStateUrl:r,origStateHeaders:C.headers,updatedStateUrl:"",updatedStateHeaders:{}},this.#B={break:this.#J,captcha:this.#z,click:this.#X,continue:this.#G,exit:this.#Y,extract:this.#Q,goto:this.#Z,hover:this.#tt,ifelse:this.#et,input:this.#rt,intercept:this.#at,loopdowhile:this.#st,loopfor:this.#it,loopineles:this.#nt,loopinstr:this.#ot,misc:this.__performMiscAction,scroll:this.#ct,select:this.#lt,setvar:this.#pt,subtask:this.#gt,wait:this.#ht},this.#V={normal:-1,parasinvalid:-2,accountinvalid:-41,accountlocked:-42,loginfailed:-43,loginneeded:-49,tologin:-40,accessforbidden:-61,cfginvalid:-69,captchafailed:-52,switchproxynow:-86,switchproxyfuture:-87,other:-98,tetimeout:-79},this.#W="",this._taskContext=this.#c,this._paras=this.#q,this.#R&&logdbg(this.#R)}catch(t){throw logerr("##te  Exeception in ScraperTask.constructor"),new Error(t)}}#dt(t){const e=this.#ft(t,"source");return"default"===e?this.#N:"browser"===e?"browser":"cheerio"===e?"cheerio":void(0,import_utils9.unreachable)(e)}#ut(t){this.#c.credits=t,logwarn(new Error)}#mt(t,e){"ignore"!==t&&logwarn(`##tmp  procErrname  eName ${t} scenario ${e} kId ${this.#p} tId ${this.#l} p ${this.#U}`);let r=-69;if("string"!=typeof t||!t||!e)throw this.#c.credits=r,new Error("##te  invalid paras in _procErrname");if("ignore"===t)return!0;throw void 0!==this.#V[t]?(r=this.#V[t],this.#c.credits=r,new Error(`##te  exit with errname ${t} in ${e}`)):(this.#c.credits=r,new Error(`##te  invalid errName ${t} in ${e}`))}_procErrname(t,e){return this.#mt(t,e)}#K(){if(this.#q.inParas&&"object"==typeof this.#q.inParas){const t=Array.from(Object.keys(this.#q.inParas)),e=t.filter(t=>t.startsWith("para")),r=t.filter(t=>t.startsWith("apara")),a=t.filter(t=>t.startsWith("rpara"));let s=e.map(t=>this.#q.inParas[t]).join(this.#L);return(r.length>0||a.length>0)&&(s=`${s}@@${r.map(t=>this.#q.inParas[t]).join(this.#L)}`,a.length>0&&(s=`${s}@@${a.map(t=>this.#q.inParas[t]).join(this.#L)}`)),s}return""}#yt(t,e=!0){return this.#c.cheerioPage?this.#c.cheerioPage.load(t,e):this.#c.cheerioPage=new import_controller2.CheerioPage(t,e),!0}#Ct(t,e){if("string"!=typeof t||"string"!=typeof e||appConfig.RESERVED_PARA_NAMES.includes(t)?this.#mt("cfginvalid","setNormalVar"):"cheerioHtml"!==t&&"cheerioXml"!==t||(this.#c.cheerioElesStack.length>0&&this.#mt("cfginvalid","cheerioElesStack"),this.#yt(e,"cheerioHtml"===t)),t.startsWith("userData.")){const r=t.slice(9);0===r.length&&this.#mt("cfginvalid","invalid userData varName"),this.#q.userData[r]=e}else if(t.startsWith("inParas_")){const r=t.slice(8);0!==r.length&&void 0!==this.#q.inParas[r]||this.#mt("cfginvalid","invalid inParas varName"),this.#q.inParas[r]=e,this.#U=this.#K()}else this.#q[t]=e;return(t.startsWith("dbgdbg")||t.startsWith("cx")&&t.match(/^cx\d$/))&&logwarn(`##dbg  setvar ${t}: ${e}`),!0}#wt(t){if("string"!=typeof t)return!1;const e=t.toLowerCase();return e.startsWith("https://")||e.startsWith("http://")}#kt(t){if(t?.[appConfig.XML_ATTR_KEY]?.type)return!0;throw new Error("##te  Invalid element type config(validate)")}#_t(t,e){if("string"!=typeof t)throw new Error(`##te  Invalid templStr: ${t}`);if(t.indexOf("${")<0)return t;try{return function(t){const e=Object.keys(t),r=Object.values(t);return new Function(...e,`return \`${this}\`;`)(...r)}.call(t,e)}catch(t){return""}}#ft(t,e,r=!0,a=!0){if(!t||"string"!=typeof e)throw new Error(`##te  Invalid config or attribute, attribute: ${e}`);const s=t[appConfig.XML_ATTR_KEY][e];if(null===s)return s;if(void 0===s){if(a)throw new Error(`##te  Config has no attribute ${e}`);return""}return!r||["type","name","version"].includes(e)?s:this.#_t(s,this.#q)}_getCfgStrAttr(t,e,r=!0,a=!0){return this.#ft(t,e,r,a)}#vt(t,e){if(!t)throw new Error(`##te  Invalid action config or attribute, attribute: ${e}`);const r=t[appConfig.XML_ATTR_KEY][e];if("boolean"!=typeof r)throw loginfo(`##te  attrNames: ${JSON.stringify(t[appConfig.XML_ATTR_KEY])}, attrName: ${e}`),new Error(`##te  Element has no bool attribute ${e} or it is undefined/null`);return r}_getCfgBoolAttr(t,e){return this.#vt(t,e)}#St(t,e,r=!1){try{const a=appConfig.XML_CHILDREN_KEY,s=appConfig.XML_ATTR_KEY;let i=t[a]?t[a][e]:null;if(Array.isArray(i))return i;if(void 0===i){const t={};return"elecontent"===e?(t[s]={type:"text",multieles:!0,trim:!0,join:"\n",line:!1,top:!1},[t]):"ocrcfg"===e?(t[s]={type:"api",nlchars:"",seperator:!1,lang:"zh"},[t]):[]}if(r)throw new Error(`##te  Invalid cfgType ${e} children of parentCfg ${JSON.stringify(t)}`);return[]}catch(t){return logerr(t),[]}}_getCfgChildrenOf(t,e,r=!1){return this.#St(t,e,r)}#bt(t,e){const r=this.#St(t,e);return r.length>0?r[0]:null}#xt(t,e){const r=this.#St(t,e);if(0===r.length)throw new Error(`##te  There is no ${e} tag config`);return r[0]}#Et(){if(this.#c.page)return this.#c.page.pageInfo();throw new Error("Cannot get pageInfo from null browser page")}#Tt(){if(this.#c.page)return this.#c.page.browserContext().browser().isHeadless();throw new Error("Cannot get headless from null browser page")}#At(t,e="ceil"){if("string"!=typeof t)throw new Error(`##te  Inavlid math expr: ${t}`);if(!["ceil","round","floor"].includes(e))throw new Error(`##te  Inavlid round type: ${e}`);return Number(getTransformExpStr(t,[{type:"tonum"},{type:e}]))}#It(t){if("string"!=typeof t)throw new Error(`##te  Inavlid bool expr: ${t}`);return Boolean(Number(getTransformExpStr(t,[{type:"tobool"}])))}async#Ot(t){let e=parseInt(t);return e>0&&await(0,import_utils9.sleep)(e),!0}#Dt(t,e=!0,r=!0,a=1,s=1){const i=t?this.#vt(t,"gen"):e,n=t?this.#vt(t,"cap"):r,o=i?a:0,c=this.#D.capName&&n?s:0;return loginfo(`##credit  addTaskCredits: gen ${o}, cap ${c}`),this.#c.credits+=(c<<appConfig.GEN_CREDITS_BITS)+o,!0}#Pt(t,e){if(t){logdbg({loginFlag:this.#vt(t,"login"),captchaFlag:this.#vt(t,"captcha"),eurl:this.#ft(t,"eurl"),eloc:this.#ft(t,"eloc"),pn1:this.#ft(t,"pn1"),pv1:this.#ft(t,"pv1"),pn2:this.#ft(t,"pn2"),pv2:this.#ft(t,"pv2"),actType:e})}}async processNewPageLoginCaptcha(t,e,r,a,s=""){const i=t.pageInfo();this.#Pt(e,s);const n="popup",o=this.#ft(e,"datapage"),c=this.#vt(e,"popupsubtask");if("popup"===o||"popuporcurrent"===o){const t=this.#p>=101?appConfig.CHECK_POPUP_PAGE_TIMEOUT1:appConfig.CHECK_POPUP_PAGE_TIMEOUT2;setTimeout(()=>this.emit(n,{code:1009,message:"timeout",data:null}),t)}await r(...a);let l=null;if("popup"===o||"popuporcurrent"===o){const[t]=await(0,import_node_events2.once)(this,n);loginfo("Taskscraper received popup event"),t&&0===t.code&&t.data?l=t.data:loginfo(`##page  kId${i.taskId} newpage popup timeout`)}return"popup"!==o||l||this.#mt("normal","noExpectedPopupPage"),l?c?this.#c.popupPage=l:this.#c.page=l:await this.processLoginAndCaptchaPage(t,"current"),this.#Pt(null,""),!0}#Rt(t,e){"browser"===t&&this.#c.page||this.#mt("cfginvalid",`notBrowserElementInAction_${e}`)}async#J(t){this.#kt(t),this.#c.loopsStack.length>0?this.#c.breakFlag=!0:logerr("##te  Error: break cannot be called outside of loop")}#$t(t){const e=this.#xt(t,"element");return this.#ft(e,"loc")}#Nt(t,e=1){const r=this.#St(t,"element");return r.length<e?(this.#mt("cfginvalid","childElements"),[]):r.map(t=>this.#ft(t,"loc"))}async#Lt(t,e,r,a){const s=this.#xt(e,"image"),i=this.#Nt(s),n=this.#bt(e,"comment"),o=n?this.#$t(n):"",c=n?this.#ft(n,"attr"):"",l=this.#xt(e,"input"),p=this.#$t(l),g=this.#vt(l,"enter"),h=g?null:this.#bt(e,"submit"),d=g||!h?"":this.#$t(h);if(!g&&!h)return this.#mt("cfginvalid","actionCaptcha"),!1;const f=this.#xt(e,"check"),u=this.#$t(f),m=this.#ft(f,"attr"),y=this.#ft(f,"failedstr"),C=this.#Ft(e),{lang:w,math:k,maxlen:_,minlen:v,question:S,space:b,texttype:x}=C,E={type:"text",imageLocs:i,commentLoc:o,commentAttr:c,inputLoc:p,enter:g,submitLoc:d,checkLoc:u,checkAttr:m,failedStr:y,case:C.case,lang:w,math:k,maxLength:parseInt(_),minLength:parseInt(v),question:S,space:b,textType:x};return await solveRecognitionCaptcha(t,E,r,a)}async#Mt(t,e,r,a){const s=this.#xt(e,"image"),i=this.#Nt(s),n=this.#bt(e,"comment"),o=n?this.#$t(n):"",c=n?this.#ft(n,"attr"):"";let l="";const p=this.#bt(e,"submit");if(p&&(l=this.#$t(p),!l))return this.#mt("cfginvalid","actionCaptcha"),!1;const g=this.#xt(e,"check"),h=this.#$t(g),d=this.#ft(g,"attr"),f=this.#ft(g,"failedstr"),u=this.#Ft(e),{lang:m,maxclicks:y,minclicks:C}=u,w={type:"coordinate",imageLocs:i,commentLoc:o,commentAttr:c,submitLoc:l,checkLoc:h,checkAttr:d,failedStr:f,lang:m,maxClicks:parseInt(y),minClicks:parseInt(C)};return await solveRecognitionCaptcha(t,w,r,a)}async#z(t){const e=this.#c.page;if(!e)return this.#mt("cfginvalid","browserPageNeeded"),!1;const r=parseInt(this.#ft(t,"try")),a=this.#xt(t,"captcha"),s=this.#ft(a,"type"),i=this.#ft(t,"errname"),n=this.#c.proxy;let o;switch(s){case"amazon":case"funcaptcha":case"geetest":case"keycaptcha":case"mtcaptcha":case"turnstile":o=await solveTokenCaptcha(e,s,n);break;case"recaptcha":const t=Number(this.#ft(a,"minscore"));o=await solveTokenCaptcha(e,s,n,{minScore:t});break;case"text":case"coordinate":o=await this.#Mt(e,a,n,r);break;case"grid":case"slider":case"rotation":o=await this.#Lt(e,a,n,r);break;default:(0,import_utils9.unreachable)(s)}return 0!==o?.code&&this.#mt(i,"actionCaptcha"),!0}async#X(t){const e=this.#c.page;if(!e)return this.#mt("cfginvalid","browserPageNeeded"),!1;this.#kt(t);const r=this.#ft(t,"type"),a=this.#ft(t,"clicktype"),s=parseInt(this.#ft(t,"try")),i=this.#ft(t,"datapage"),n=this.#vt(t,"navigate"),o=this.#ft(t,"errname"),c=this.#St(t,"element",!0),{elementSource:l,elements:p}=await this.#jt(c,s);if(this.#Rt(l,"click"),0===p.length)return loginfo(`## te  ----  no element to click, actCfg: ${JSON.stringify(t).slice(0,200)}`),this.#mt(o,"actionClick"),!1;const g=p[0];try{logdbg(`##te  click text: ${await g.innerText()}`)}catch(t){logdbg(t)}e.browserContext().browser().isHeadless()||(logdbg(`##te  pageId${e.id()} bringToFront in click`),await e.bringToFront());const h=this.#ft(t,"downloadpath"),d=appConfig.cfg.baseDownloadDir,f=path4.join(d,h);if(h&&d){await(0,import_utils9.fsCheckOrCreateSubdir)(d,h,"")?await e.sendCDPMessage("Page.setDownloadBehavior",{behavior:"allow",downloadPath:f}):this.#mt("other","actionClickDwlRelDir")}if("click"===a||"evaluate"===a?await this.processNewPageLoginCaptcha(e,t,async t=>{await t.click({clickType:a})},[g],r):(logerr(`##task invalid clicktype ${a} in performClickAction`),this.#mt("cfginvalid","actionClick")),"current"===i&&n)try{await(0,import_utils9.sleep)(5e3)}catch(t){}const u=this.#ft(t,"wait");await this.#Ot(u);const m=this.#ft(t,"filename");let y="";if(f&&m){y=path4.join(f,m);let t=0;const e=300;for(t=0;t<e;t++){if(await(0,import_utils9.fileExist)(y))break;await(0,import_utils9.sleep)(1e3)}t>=e&&(logwarn(`##te  download  kId ${this.#p}: failed to download file ${y}`),this.#mt("other","actionClickDwlTimeout"))}const C=this.#ft(t,"pathvarname");return C&&this.#Ct(C,y),this.#Dt(t),!0}async#G(t){return this.#kt(t),this.#c.loopsStack.length>0?(this.#c.continueFlag=!0,!0):(logerr("##te  Error: continue cannot be called outside of loop"),!1)}async#Y(t){this.#kt(t);const e=this.#ft(t,"errname");if(!e)return!1;const r=void 0!==this.#V[e]?this.#V[e]:-69;throw this.#c.credits=r,new Error(`##te  action_throw with errname ${e}`)}#qt(t){try{const e=this.#c.page,r=e?e.url():this.#q.pageUrl?this.#q.pageUrl:"";if(!r)return logwarn("Please execute page.goto or set variable pageUrl first"),t;const a=new URL(t,r);return a?a.href:t}catch(e){return t}}#Ut(t,e=""){if(!t)return null;try{return e?new RegExp(t,e):new RegExp(t)}catch(t){return this.#mt("cfginvalid","getRegExp"),null}}async#Ht(t,e){let r="text";t&&(r=this.#ft(t,"type"));let a="step00",s="",i=!0,n=!1,o="\n",c=!1,l=!1,p=!1,g=[];try{if(e.length>0){let h=e[0];switch(r){case"text":a="step11",t&&(i=this.#vt(t,"multieles"),n=this.#vt(t,"trim"),o=this.#ft(t,"join"),c=this.#vt(t,"line"),l=this.#vt(t,"top")),a="step12";for(let t of e){if(s=await t.innerText(l),n&&(s=s.trim()),!i)break;g.push(s)}a="step13",i&&(s="stringify"===o?JSON.stringify(g):g.join(o)),c&&(s=s.replace(/[\s\r\n]+/g," ")),a="step14";break;case"attr":a="step21";const d=this.#ft(t,"attrname"),f=this.#ft(t,"attrname2"),u=this.#ft(t,"attrname3"),m=this.#ft(t,"attrname4");n=this.#vt(t,"trim");const y=this.#ft(t,"pattern"),C=this.#vt(t,"boolattr");i=this.#vt(t,"multieles"),o=this.#ft(t,"join"),c=this.#vt(t,"line"),p=this.#vt(t,"absolute"),a="step22";for(let t of e){if(C)a="step22",s=await t.hasAttribute(d)?"1":"0";else{a="step23";const e=this.#Ut(y);for(const r of[d,f,u,m]){if(!r)break;let a=await t.attribute(r);if(s="string"==typeof a?a:null===a?"":String(a),n&&(s=s.trim()),!e)break;if(s.match(e))break;s=""}p&&(s=this.#qt(s))}if(!i)break;g.push(s),a="step24"}i&&(s="stringify"===o?JSON.stringify(g):g.join(o)),c&&(s=s.replace(/[\s\r\n]+/g," ")),a="step25";break;case"innerhtml":s=await h.innerHtml();break;case"length":s=String(e.length);break;case"outerhtml":s=await h.outerHtml();break;case"pictext":s="pictext";break;case"image":s="image";break;case"query":s="query";break;default:throw new Error(`##te  Invalid content type: ${r}`)}}else"length"===r&&(s="0");return s}catch(i){throw loginfo(`##te  processBrowserElecontentCfg: tId${this.#l} ecCfg: ${JSON.stringify(t).slice(0,200)}`),loginfo(`##te  err: ${JSON.stringify(i)}`),loginfo(`##te  contentType ${r}, content ${s}, onelineFlag ${c}`),loginfo(`##te  message ${a}`),loginfo(`##te  $browserEles ${e}, length ${e.length}, e0 ${e[0]}`),new Error(i)}}#Bt(t,e){try{let r="";const a=this.#ft(t,"defaultval"),s=this.#vt(t,"usevar");let i=this.#St(t,"fun",!0).map(t=>t[appConfig.XML_ATTR_KEY]);if(s){const t=i;i=[];for(const e of t){const t=Object.assign({},e);for(const e of Object.keys(t))"string"==typeof t[e]&&t[e].includes("${")&&(t[e]=this.#_t(t[e],this.#q));i.push(t)}}return r=getTransformExpStr(e,i,a),r}catch(e){throw loginfo(`##te  processTransformCfg: tId${this.#l} tCfg: ${JSON.stringify(t).slice(0,200)}`),loginfo(`##te  err: ${JSON.stringify(e)}`),new Error(e)}}async#Vt(t){if(t){if(!this.#c.popupPage)throw new Error("No valid popup page");return this.#c.popupPage}if(this.#c.page){const t=await this.#c.page.browserContext().getPage(!0);if(t)return t;throw new Error("No new page to perform sync subtask")}throw new Error("logic error in #getSubtaskPage")}async#Wt(t){return await t.free(),!0}#Kt(t){const{proxy:e,browserApiContext:r,standaloneApiContext:a}=this.#c;return{proxy:e,page:t,browserApiContext:r,standaloneApiContext:a}}async#Jt(t){try{let e="";const r=parseInt(this.#ft(t,"tid")),a=parseInt(this.#ft(t,"minlen")),s=this.#ft(t,"parasstr");let i=this.#ft(t,"errname");const n=this.#vt(t,"popuppage");if(n&&!this.#c.popupPage){const t="ignore"===i?"cfginvalid":i;this.#mt(t,"subtaskSynMemNopage")}if(s.length>=a){const t=await this.#Vt(n),{taskData:a}=await this.#j(r,s,this.#Kt(t),"memSync");await this.#Wt(t);const{credits:i,execData:o}=a;if(n&&(this.#c.popupPage=null),logdbg(`##te  memsyncsubtask t${r} p${s} c${i} data: ${JSON.stringify(o)}`),i>=0){if(o&&1===Object.keys(o).length&&Array.isArray(Object.values(o)[0])){const t=Object.values(o)[0];if(Array.isArray(t)&&1===t.length&&1===Object.keys(t[0]).length){const r=Object.keys(t[0])[0];e=t[0][r]}else e=JSON.stringify(o)}else e=JSON.stringify(o);let t=calcGenCredits(i),r=calcCapCredits(i);this.#Dt(null,!0,!0,t,r)}}else this.#mt(i,"subtaskSynMemFailed");return e}catch(e){throw loginfo(`##te  processSyncSubtask: tId${this.#l} pCfg: ${JSON.stringify(t).slice(0,200)}`),loginfo(`##te  err: ${JSON.stringify(e)}`),new Error(e)}}#zt(t,e){const r={browserListQuery:t.browserlist?t.browerlist:"",browsers:["chrome"],operatingSystems:["windows"],devices:["desktop"],httpVersion:Number(t.httpversion)>0?t.httpversion:"2"},a=this.#St(e,"httpbrowser");if(a.length>0){const t=[];a.forEach(e=>{const r=this.#ft(e,"name");r||this.#mt("cfginvalid","actGetHttpHeaders");const a={name:r},s=parseInt(this.#ft(e,"minversion"));s>0&&(a.minVersion=s);const i=parseInt(this.#ft(e,"maxversion"));i>0&&(a.maxVersion=i);const n=this.#ft(e,"httpversion");n&&Number(n)>0&&(a.httpVersion=n),t.push(a)}),r.browsers=t}const s=this.#St(e,"httpos");if(s.length>0){const t=[];s.forEach(e=>{const r=this.#ft(e,"os");r||this.#mt("cfginvalid","actGetHttpHeaders"),t.push(r)}),r.operatingSystems=t}const i=this.#St(e,"httpdevice");if(i.length>0){const t=[];i.forEach(e=>{const r=this.#ft(e,"device");r||this.#mt("cfginvalid","actGetHttpHeaders"),t.push(r)}),r.devices=t}return r}async#Xt(t,e){try{let r="",a=[];if("element"===t){const t=this.#ft(e,"type"),s=this.#St(e,"element",!0),i=this.#xt(e,"elecontent"),n=this.#ft(e,"try",!0,!1),o=n?parseInt(n):1,{elements:c}=await this.#jt(s,o);if(a=c,0===a.length){if("length"===this.#ft(i,"type"))r="0";else if("element"===t&&e[appConfig.XML_ATTR_KEY].colname){const t=this.#ft(e,"errname");this.#mt(t,"columnElement")}}else r=await this.#Ht(i,c)}else if("templstr"===t){const t=this.#xt(e,"templstr");r=this.#ft(t,"templ")}else if("get"===t){const t=this.#xt(e,"get"),a=this.#Ft(t);let s=this.#c.page,i="page";const n={paras:this.#q},o=this.#ft(t,"type");if("screenshot"===o){const e=this.#bt(t,"element");if(e){const{elements:t}=await this.#jt([e],1);if(!(t.length>0))throw new Error("##te no elements selected in get_screenshot");i="element",r=await GetPara.getParaInElement(a,t[0],n)}}else if("httpheaders"===o){const e=a.headermethod;n.interceptionData=this.#H,"genNewByHttpOptions"===e&&(n.genHeaderOptions=this.#zt(a,t));const r=[];this.#St(t,"httpheader",!0).forEach(t=>{let e=this.#ft(t,"name");if(e=e.toLowerCase().trim(),e){const a=this.#ft(t,"source"),s=this.#ft(t,"value"),i=this.#vt(t,"delete");r.push({name:e,source:a,value:s,deleteFlag:i})}}),n.newHttpHeaders=r}else if("api"===o){n.interceptionData=this.#H,n.proxyUrl=this.#c.proxy?this.#c.proxy.proxyUrl:"";const t=a.context;let e=!1;if("browser"===t)n.apiContext=this.#c.browserApiContext,e=!0;else if("standalone"===t)n.apiContext=this.#c.standaloneApiContext,e=!0;else if("state"===t)n.apiContext=this.#c.stateApiContext,e=!0;else if("task"===t){if(!this.#c.taskApiContext){const t=this.#c.proxy?{proxy:this.#c.proxy}:{};this.#c.taskApiContext=await import_controller2.controller.newApiContext(t)}n.taskApiContext=this.#c.taskApiContext,e=!0}e&&!n.apiContext&&this.#mt("cfginvalid","getPara_nullApiContext")}else"response"===o&&(n.interceptionData=this.#H);s&&"page"===i&&["content","cookies","mhtml","pdf","screenshot","title","window"].includes(a.type)?r=await GetPara.getParaInPage(a,s,n):["api","addedtaskparas","datetime","execedtaskparas","file","httpheaders","queuedcntwithparas","queuedtaskparas","random","response","special","sysparas"].includes(a.type)?r=await GetPara.getPara(a,n):["screenshot"].includes(a.type)&&"element"===i||this.#mt("cfginvalid","getPara")}else if("subtask"===t){const t=this.#xt(e,"subtask");loginfo(`##te  to execsync subtaskCfg: ${JSON.stringify(t)}`),r=await this.#Jt(t)}else{if("ocr"!==t)throw new Error(`##te  Invalid content source type: ${t}`);{let t=this.#xt(e,"ocrcfg");const a=this.#St(e,"element",!0),{elementSource:s,elements:i}=await this.#jt(a,1,!1);if(i.length>0&&(this.#Rt(s,"ocr"),r=await this.__processBrowserOcrCfg(t,i),"number"==typeof r&&r<0))throw new Error(`##te  Error with ocr content: ${r}`)}}const s=this.#bt(e,"decodefontsvg"),i=this.#bt(e,"decodefontttf");(s||i)&&(r=await this.__decodeFontInContent(r,s,i));const n=this.#bt(e,"transform");return n&&(r=this.#Bt(n,r),null===r&&this.#mt("cfginvalid","invalidFontchars")),{content:r,elements:a}}catch(t){throw loginfo(`##te  getEleOrTemplTransform: tId${this.#l} pCfg: ${JSON.stringify(e).slice(0,200)}`),loginfo(`##te  err: ${JSON.stringify(t)}`),new Error(t)}}#Gt(t,e){let r="";if(this.#c.execData[t]&&this.#c.execData[t].length>0){let a=this.#c.execData[t].length,s=this.#c.execData[t][a-1][e];r=String("boolean"==typeof s?Number(s):s)}return r}async#Yt(t,e){const r=this.#ft(t,"valerrname"),a=this.#ft(t,"pattern");try{this.#kt(t);const s=this.#ft(t,"type"),i=this.#ft(t,"colname");let{content:n}=await this.#Xt(s,t);if(0===n.length){this.#vt(t,"completed")&&(n=this.#Gt(e,i))}if("ignore"!==r&&a){const e=this.#ft(t,"flags"),s=this.#Ut(a,e);s&&!n.match(s)&&this.#mt(r,"columnContent")}this.#vt(t,"setvar")&&this.#Ct(i,n);const o=this.#vt(t,"discarded"),c={};if(!o){let e;const r=this.#ft(t,"datatype");"number"===r?e=this.#At(n,"round"):"boolen"===r&&(e=this.#It(n)),c[i]="string"===r?n:e}return c}catch(e){throw this.#c.credits>=0&&(loginfo(`##te  processColumnCfg: tId${this.#l}, fCfg ${JSON.stringify(t).slice(0,100)}`),loginfo(`##te  err: ${JSON.stringify(e)}`)),new Error(e)}}async#Q(t){this.#kt(t);const e=this.#ft(t,"tabname");if("string"!=typeof e||0===e.length)throw new Error(`##te  Invalid table name: ${e}`);this.#c.execData[e]||(this.#c.execData[e]=[]);const r={},a=this.#St(t,"column",!0);for(let t of a){const a=this.#ft(t,"colname");if(!(this.#M.length>0)||this.#M.includes(a))try{let s=await this.#Yt(t,e);Object.assign(r,s),a.startsWith("cx")&&loginfo(`##dbg  extract: ${a}=${r[a]}`)}catch(e){throw this.#c.credits>=0&&loginfo(`##te  extract column tId${this.#l} fCfg: ${JSON.stringify(t).slice(0,200)}`),e}}return this.#c.execData[e].push(r),!0}#Qt(t){try{const e=JSON.parse(t);return e&&"object"==typeof e?e:null}catch(t){this.#mt("cfginvalid","actionGotoHeaders")}}async#Z(t){const e=this.#c.page;if(!e)return this.#mt("cfginvalid","browserPageNeeded"),!1;this.#c.browserElesStack.length>0&&this.#mt("cfginvalid","inLoopOfBrowserElements");let r="";try{this.#kt(t);const a=this.#ft(t,"type");r=this.#ft(t,"url");const s=this.#ft(t,"wait"),i=this.#vt(t,"reuse"),n=this.#vt(t,"encodeuri"),o=this.#ft(t,"headers"),c=this.#ft(t,"referer");if(n&&(r=encodeURI(r)),!this.#wt(r))throw new Error(`##te  Invalid url: ${r} when trying to open this page`);if(loginfo(`##te  goto url ${r}`),i){if(e.url().includes(r))return await this.#Ot(s),this.#Dt(t),e}if(this.#W=r,o.trim()){const t=this.#Qt(o.trim());t&&await e.setExtraHTTPHeaders(t)}return await this.processNewPageLoginCaptcha(e,t,async(t,e,r)=>{r?await t.goto(e,{waitUntil:"load",timeout:0,referer:r}):await t.goto(e,{waitUntil:"load",timeout:0})},[e,r,c],a),await this.#Ot(s),this.#Dt(t),e}catch(t){if(loginfo(`##te  kId${this.#p} goto: ${r}`),-46!==this.#c.credits&&(logerr(`##te  kId${this.#p} error credits ${this.#c.credits} in performGotoAction`),logerr(t)),this.#c.credits>=0){const e=t.message?t.message:"";if(e.includes("net::ERR_INTERNET_DISCONNECTED"))this.#ut(-71);else if(e.includes("net::ERR_NAME_NOT_RESOLVED")){await(0,import_utils9.checkDns)(appConfig.CHECK_DNS_DOMAIN)?this.#ut(-69):this.#ut(-71)}else e.includes("net::ERR_CONNECTION_TIMED_OUT")?this.#ut(-72):e.includes("net::ERR_CONNECTION_REFUSED")||e.includes("net::ERR_CONNECTION_RESET")?this.#ut(-98):e.includes("net::ERR_PROXY_CONNECTION_FAILED ")||e.includes("net::ERR_EMPTY_RESPONSE")||e.includes("net::ERR_TIMED_OUT")||e.includes("net::ERR_TUNNEL_CONNECTION_FAILED")?this.#ut(-86):e.includes("net::ERR_INVALID_AUTH_CREDENTIALS")?this.#ut(-88):(e.includes("Navigation timeout"),this.#ut(-85))}throw new Error(t)}}async#Zt(t,e=10){let r=null;for(let a=0;a<e;a++)try{return await t.hover(),!0}catch(t){loginfo(t),await(0,import_utils9.sleep)(2e3),r=t}throw r}async#tt(t){const e=this.#c.page;if(!e)return this.#mt("cfginvalid","browserPageNeeded"),!1;this.#kt(t);const r=parseInt(this.#ft(t,"try")),a=this.#ft(t,"errname");try{const s=this.#St(t,"element",!0),{elementSource:i,elements:n}=await this.#jt(s,r);if(this.#Rt(i,"hover"),0===n.length)return this.#mt(a,"actionHover"),e;this.#Tt()&&await e.bringToFront(),logdbg(`##te  pageIdx${this.#Et().pageIdx} hover`),await this.#Zt(n[0]);const o=this.#ft(t,"wait");return await this.#Ot(o),this.#Dt(t),e}catch(t){return loginfo(t),this.#mt("other","actionHover"),e}}async#te(t){this.#kt(t);const e=this.#ft(t,"type");let r=!1;if(["element","templstr"].includes(e)){let a="";a=(await this.#Xt(e,t)).content,r=this.#It(a)}else{if("else"!==e)throw new Error("##te  Invalid type(condition)");r=!0}if(r){const e=this.#St(t,"action");await this.#ee(e)}return r}async#et(t){this.#kt(t);const e=this.#St(t,"condition");for(let t of e)if(await this.#te(t))break;return!0}async#rt(t){if(!this.#c.page)return this.#mt("cfginvalid","browserPageNeeded"),!1;this.#kt(t);const e=this.#ft(t,"content"),r=this.#vt(t,"replace"),a=this.#vt(t,"enter"),s=parseInt(this.#ft(t,"try")),i=this.#ft(t,"errname"),n=this.#St(t,"element",!0);try{const{elementSource:o,elements:c}=await this.#jt(n,s);if(this.#Rt(o,"input"),0===c.length)return this.#mt(i,"actionInput"),!1;await c[0].input(e,{replace:r,enter:a});const l=this.#ft(t,"wait");return await this.#Ot(l),this.#Dt(t),!0}catch(t){return logerr(t),!1}}#Ft(t,e=!0){const r=Object.assign({},t[appConfig.XML_ATTR_KEY]);if(!r||"object"!=typeof r)throw new Error("Invalid origCfg in transformStrAttrsInCfg");if(!e)return r;for(const t of Object.keys(r))"string"==typeof r[t]&&r[t].includes("${")&&(r[t]=this.#_t(r[t],this.#q));return r}#re(t){const e=[];for(const r of t){const{type:t,method:a,url:s,resourcetype:i,postdata:n}=r,o={};if(a&&"all"!==a&&(o.methods=[a.toUpperCase()]),s&&(o.url=new RegExp(s)),i&&(o.resourceTypes=[i]),n&&(o.postData=new RegExp(n)),"abort"===t){const t={requestMatch:o,action:"abort"};e.push(t)}else{if("fufill"!==t)throw new Error(`Invalid action type ${t}`);{const{fulfill:t}=r,a={requestMatch:o,action:"fulfill",fulfill:t};e.push(a)}}}return e}#ae(t){const e=[];for(const r of t){const{type:t,method:a,url:s,resourcetype:i,postdata:n,minsize:o,maxsize:c}=r,l={};a&&"all"!==a&&(l.methods=[a.toUpperCase()]),s&&(l.url=new RegExp(s)),i&&(l.resourceTypes=[i]),n&&(l.postData=new RegExp(n));const p={},g=parseInt(o);g>0&&(p.minLength=g);const h=parseInt(c);if(h>0&&(p.maxLength=h),"cache"===t){const{tabname:t,contenttype:a,responses:s,requestheaders:i,responseheaders:n}=r,o={requestMatch:l,responseMatch:p,handler:responseCacheCb,handlerOptions:{execData:this.#c.execData,contentType:a,tabname:t,responseFlag:s,requestHeadersFlag:i,responseHeadersFlag:n,interceptionData:this.#H}};e.push(o)}else{if("save"!==t)throw new Error(`Invalid response type ${t}`);{const{basedir:t,encoding:a,extname:s,hashmethod:i,hostnameparts:n,params:o,pathtype:c,tabname:g,validextnames:h}=r,d={requestMatch:l,responseMatch:p,handler:responseSaveCb,handlerOptions:{execData:this.#c.execData,baseDir:t,pathType:c,hashMethod:i,hostnameParts:parseInt(n),extname:s,validExtnames:h,params:o,encoding:a,tabname:g}};e.push(d)}}}return e}async#at(t){const e=this.#c.page;if(!e)return this.#mt("cfginvalid","browserPageNeeded"),!1;this.#kt(t);const r=this.#ft(t,"subtype");let a=this.#St(t,"request"),s=this.#St(t,"response");try{if(a=a.map(t=>this.#Ft(t)),s=s.map(t=>this.#Ft(t)),"set"===r){if(a.length<=0&&s.length<=0&&this.#mt("cfginvalid","actionIntercept"),a.length>0){const t=this.#re(a);await e.setRequestInterception(t)}if(s.length>0){const t=this.#ae(s);await e.setResponseInterception(t)}}else"clear"===r&&(await e.clearRequestInterceptions(),await e.clearResponseInterceptions());return!0}catch(t){return logerr(t),!1}}async#it(t){this.#kt(t);try{const e=this.#ft(t,"errname");let r=this.#ft(t,"roundtype"),a=this.#At(this.#ft(t,"from"),r),s=this.#At(this.#ft(t,"to"),r),i=this.#At(this.#ft(t,"step"),r),n=this.#ft(t,"varname"),o=this.#At(this.#ft(t,"maxloops"),r);if(o<0)return!1;let c=0;for(let e=a;e<=s&&!(o&&c++>=o);e+=i){this.#c.loopsStack.push(e),this.#Ct(n,String(e));const r=this.#St(t,"action");if(await this.#ee(r),this.#c.loopsStack.pop(),this.#c.continueFlag&&(this.#c.continueFlag=!1),this.#c.breakFlag){this.#c.breakFlag=!1;break}}return c<1&&this.#mt(e,"actionLoopfor"),!0}catch(t){throw t}}async#se(t){try{const e=this.#ft(t,"loc").trim(),r=this.#dt(t),a=this.#vt(t,"absolute"),s=this.#St(t,"iframe",!1).map(t=>this.#Ft(t)).map(t=>{const{srcprefix:e,loc:r}=t;return{src:e,selector:r}}),i=this.#c.page,n={elementSource:r,elements:[]};if("browser"===r){if(!i)throw new Error("Cannot get elements from null browser page");if(0===this.#c.browserElesStack.length)n.elements=await i.findElements(e,s);else{const t=this.#c.browserElesStack.slice(-1)[0];n.elements=await t.findElements(e,s,a)}return n}if("cheerio"===r){if(s.length>0)throw new Error("Iframe config is invalid in cheerio source");let t=this.#c.cheerioPage;if(!t){if(!i)throw new Error("Please set CheerioPage first, from which to select elememnts");const e=await i.content();e.length>10&&(this.#yt(e,!0),t=this.#c.cheerioPage)}if(!t)throw new Error("Cannot get elements from null cheerioPage");if(0===this.#c.cheerioElesStack.length)n.elements=await t.findElements(e);else{const t=this.#c.cheerioElesStack.slice(-1)[0];n.elements=await t.findElements(e,s,a)}return n}(0,import_utils9.unreachable)(r)}catch(e){throw loginfo(`##te  processElementCfg: tId${this.#l} eleCfg: ${JSON.stringify(t).slice(0,200)}`),new Error(e)}}async#jt(t,e=1,r=!0){try{if(0===t.length)throw new Error("No element tags");const a={elementSource:"browser",elements:[]};for(let s=0;s<e;s++){let e="";const i=[];for(const a of t){const{elementSource:t,elements:s}=await this.#se(a);if(e&&e!=t)throw new Error("Element sources of all element tags must be the same");if(e=t,s.length>0&&(s.forEach(t=>i.push(t)),r))break}if(e&&(a.elementSource=e),i.length>0)return a.elements=i,a;s>0&&(loginfo(`##tmp ${s} try failed in getEleOrTemplTransform`),await(0,import_utils9.sleep)(1e3))}return a}catch(e){throw loginfo(`##te  processElementCfgs: eleCfgs: ${JSON.stringify(t).slice(0,200)}`),new Error(e)}}async#nt(t){if(!this.#c.page)return this.#mt("cfginvalid","browserPageNeeded"),!1;this.#kt(t);const e=this.#ft(t,"varname"),r=e.startsWith("position"),a=this.#ft(t,"errname"),s=this.#St(t,"element",!0);let i=this.#At(this.#ft(t,"maxloops"),"round"),n=this.#At(this.#ft(t,"start"),"round"),o=this.#At(this.#ft(t,"end"),"round"),c=this.#At(this.#ft(t,"step"),"round");if(c<1&&(c=1),i<0)return!1;const{elementSource:l,elements:p}=await this.#jt(s,1),g=p.length;if(0===g)return this.#mt(a,"actionLoopineles"),!0;n<=0&&(n+=g),o<=0?o+=g:o>g&&(o=g);let h=0;for(let a=n-1;a<o&&!(i&&h>=i);a+=c){e&&this.#Ct(e,String(r?a+1:a));const s=p[a];"browser"===l?this.#c.browserElesStack.push(s):"cheerio"===l?this.#c.cheerioElesStack.push(s):(0,import_utils9.unreachable)(l),this.#c.loopsStack.push(s);const i=this.#St(t,"action");if(await this.#ee(i),this.#c.loopsStack.pop(),"browser"===l?this.#c.browserElesStack.pop():"cheerio"===l?this.#c.cheerioElesStack.pop():(0,import_utils9.unreachable)(l),h++,this.#c.continueFlag&&(this.#c.continueFlag=!1),this.#c.breakFlag){this.#c.breakFlag=!1;break}}return!0}async#ot(t){this.#kt(t);let e=this.#ft(t,"split");const r=this.#vt(t,"trim"),a=this.#ft(t,"errname");let s;try{s=new RegExp(e)}catch(t){throw new Error(`##te  Invalid regexp: ${s}`)}const i=r?this.#ft(t,"list").trim().split(s).map(t=>t.trim()).filter(t=>t.length>0):this.#ft(t,"list").split(s),n=this.#ft(t,"varname");let o=this.#At(this.#ft(t,"maxloops"),"round");if(o<0)return!1;let c=0;for(const e of i){if(o&&c++>=o)break;this.#c.loopsStack.push(e),this.#Ct(n,e);const r=this.#St(t,"action");if(await this.#ee(r),this.#c.loopsStack.pop(),this.#c.continueFlag&&(this.#c.continueFlag=!1),this.#c.breakFlag){this.#c.breakFlag=!1;break}}return c<1&&this.#mt(a,"actionLoopinstr"),!0}async#st(t){this.#kt(t);const e=this.#ft(t,"subtype"),r=this.#vt(t,"iswhile"),a=this.#ft(t,"varname"),s="element"===e&&this.#vt(t,"click"),i=(this.#vt(t,"navigate"),"element"===e&&this.#vt(t,"gen")),n="element"===e&&this.#vt(t,"cap"),o=s?this.#ft(t,"errname"):"ignore",c=s?this.#ft(t,"wait"):"-1";let l=this.#At(this.#ft(t,"maxloops"),"round");if(l<0)return!1;let{content:p,elements:g}=await this.#Xt(e,t),h=0;for(;0===h&&!r||this.#It(p);){a&&this.#Ct(a,String(h)),"element"!==e||!s||this.#F.includes("click")||this.#F.includes("operate")||(g.length<=0?this.#mt(o,"actionLoopdowhileElement"):(await g[0].click(),this.#Dt(t,i,n))),this.#c.loopsStack.push(h);const r=this.#St(t,"action");if(await this.#ee(r),this.#c.loopsStack.pop(),this.#c.continueFlag&&(this.#c.continueFlag=!1),this.#c.breakFlag){this.#c.breakFlag=!1;break}if(await this.#Ot(c),h++,l&&h>=l)break;let d=await this.#Xt(e,t);p=d.content,g=d.elements}return!0}async#ie(t){const e=this.#ft(t,"cookies"),r=this.#ft(t,"localstorage"),a=this.#ft(t,"headers"),s=this.#ft(t,"userdata"),{cookies:i,localStorage:n,userData:o}=this.#c.origStateData,c={cookies:[],localStorage:[],headers:{},userData:{}};let l;if("orig"===e)c.cookies=i;else if("new"===e)if(this.#c.page)c.cookies=await this.#c.page.cookies();else{if(!this.#c.standaloneApiContext)throw new Error(`##templ ${this.#l} has no page when getting stateData of page`);l||(l=await this.#c.standaloneApiContext.stateData()),c.cookies=l.cookies}if("orig"===r)c.localStorage=n;else if("new"===r)if(this.#c.page)c.localStorage=await this.#c.page.localStroage();else{if(!this.#c.standaloneApiContext)throw new Error(`##templ ${this.#l} has no page when getting stateData of page`);l||(l=await this.#c.standaloneApiContext.stateData()),c.localStorage=l.localStorage}const p=this.#H;p[a]&&(c.headers=Object.assign({},p[a])),"orig"===s?c.userData=o:"new"===s&&(c.userData=Object.assign({},this.#q.userData)),this.#c.newStateData=c}async _performGetstatedataInMisc(t){return await this.#ie(t)}async#ct(t){const e=this.#c.page;if(!e)return this.#mt("cfginvalid","browserPageNeeded"),!1;this.#kt(t);const r=this.#ft(t,"subtype");let a=["by","to"].includes(r)?parseInt(this.#ft(t,"height")):0;const s=["by","to"].includes(r)?this.#ft(t,"unit"):"none";let i=0,n="";"bodypct"===s?(n=await e.windowMember(["document","body","scrollHeight"]),i=parseInt(n)):"screenpct"===s?n=await e.windowMember(["screen","height"]):"windowpct"===s&&(n=await e.windowMember(["innerHeight"]));const o=n?parseInt(n):1;if(["bodypct","screenpct","windowpct"].includes(s)){if("number"!=typeof o||o<=0)throw new Error("##te  invalid unitInt in performScrollAction");a=Math.floor(a*o/100)}let c="by"===r?parseInt(this.#ft(t,"maxtimes")):0;const l="by"===r?parseInt(this.#ft(t,"interval")):0;if("by"===r){if(0===c){if(i<=0){const t=await e.windowMember(["document","body","scrollHeight"]);i=parseInt(t)}c=Math.ceil(i/a-1)}else c>200?c=200:c<0&&(c=10);for(let r=0;r<c;r++){await e.scrollBy(0,a),await(0,import_utils9.sleep)(l);const r=this.#St(t,"element");if(r&&r.length>0){const{elementSource:t,elements:e}=await this.#jt(r,1);if(this.#Rt(t,"scrollBy"),e.length>0)break}}}else if("to"===r)await e.scrollTo(0,a);else if("intoview"===r){const e=this.#St(t,"element",!0),{elementSource:r,elements:a}=await this.#jt(e,1);if(this.#Rt(r,"scrollIntoView"),a.length<=0){const e=this.#ft(t,"errname");return this.#mt(e,"actionScrollIntoview"),!1}await a[0].scrollIntoView()}return this.#Dt(t),!0}async#ne(t,e,r,a,s=1){if(t.length<=0)throw new Error("##te  No element cfg in select action");const{elementSource:i,elements:n}=await this.#jt(t,s);if(this.#Rt(i,"select"),n.length<=0)return this.#mt(a,"actionSelect"),!1;const o={type:"value"},c=r.split(",");return"index"===e?(o.type,o.indexes=c.map(t=>parseInt(t))):"label"===e?(o.type,o.labels=c):"value"===e?(o.type,o.values=c):(0,import_utils9.unreachable)(e),await n[0].select(o),!0}async#lt(t){const e=this.#c.page;if(!e)return this.#mt("cfginvalid","browserPageNeeded"),!1;e.title,this.#kt(t);const r=this.#ft(t,"selecttype"),a=this.#ft(t,"selectval"),s=parseInt(this.#ft(t,"try")),i=this.#ft(t,"errname"),n=this.#St(t,"element",!0);await this.#ne(n,r,a,i,s);const o=this.#ft(t,"wait");return await this.#Ot(o),this.#Dt(t),!0}async#pt(t){this.#kt(t);const e=this.#ft(t,"subtype"),r=e.slice(0,1).toUpperCase()+e.slice(1),a=this.#ft(t,"varname"),s=this.#ft(t,"valerrname"),i=this.#ft(t,"pattern"),n=this.#ft(t,"defaultval");let o=this.#ft(t,"path"),c="";if(["element","get","templstr","subtask","ocr"].includes(e))try{c=(await this.#Xt(e,t)).content}catch(t){loginfo(t),c=""}else if("dbquery"===e)c=await this.__processDbquery(t,n);else{if("file"!==e)throw new Error(`##te  Invalid subtupe ${e} in setvar action`);try{const e=this.#xt(t,"file");let s=this.#ft(e,"path");const i=this.#ft(e,"encoding"),n=this.#vt(t,"proxy"),o=s.toLocaleLowerCase();if(o.startsWith("http://")||o.startsWith("https://")){const t=n&&this.#c.proxy?this.#c.proxy.proxyUrl:"";c=(await lsdDownload(s,t)).toString(i)}else path4.isAbsolute(s)?appConfig.cfg.inputFileDirPart&&!path4.dirname(s).split(path4.sep).includes(appConfig.cfg.inputFileDirPart)&&this.#mt("normal",`actionSetvar${r} ${a}`):s=path4.join(appConfig.cfg.baseFileDir,s),c=fs4.readFileSync(s,{encoding:i})}catch(t){logerr(t),c=""}}if("ignore"!==s&&i){const e=this.#ft(t,"flags"),r=this.#Ut(i,e);r&&!c.match(r)&&this.#mt(s,"setvarContent")}return!c&&n&&(c=n),this.#Ct(a,c),o&&(o=path4.join(appConfig.cfg.baseFileDir,o),fs4.writeFileSync(o,c)),!0}async#gt(t){this.#kt(t);const e=this.#ft(t,"subtasks"),r=this.#vt(t,"sameasparent");if(e&&e.startsWith("[{"))try{const t=JSON.parse(e);if(Array.isArray(t)&&t.every(t=>t.tid>0&&"string"==typeof t.parasstr))for(const e of t){if(this.#c.subtasks.find(t=>t.tid===e.tid&&t.parasstr===e.parasstr)||e.minlen>0&&e.parasstr.length<e.minlen)continue;const t={tid:e.tid,parasstr:e.parasstr};e.idx>0&&(t.idx=e.idx),r&&(t.sapFlag=!0),this.#c.subtasks.push(t)}else t.length>=1&&("string"==typeof t[0].parasstr&&"number"==typeof t[0].tid||logerr(`##te invalid subtask: ${JSON.stringify(t[0])} in performSubtaskAction`))}catch(t){logerr(t)}const a=this.#St(t,"subtask");for(const t of a){const e=parseInt(this.#ft(t,"tid")),a=parseInt(this.#ft(t,"minlen")),s=this.#ft(t,"parasstr"),i=this.#ft(t,"idx").trim(),n=i.length>0&&i.match(/^[0-9]+$/)?parseInt(i):0,o=this.#ft(t,"errname"),c=this.#vt(t,"popuppage");if(c&&!this.#c.popupPage){const t="ignore"===o?"cfginvalid":o;this.#mt(t,"subtaskSyncIndepNopage")}if(s.length>=a)if(c){const t=await this.#Vt(c),r=await this.#j(e,s,this.#Kt(t),"indSync");await this.#Wt(t),this.#c.popupPage=null;const{credits:a,execData:i}=r.taskData;a<-1?this.#mt(o,"subtaskSyncIndepFailed"):(this.#c.subtaskResults.push(r),logdbg(`##te  indsyncsubtask t${e} p${s} c${a} data: ${JSON.stringify(i)}`))}else{if(!this.#c.subtasks.find(t=>t.tid===e&&t.parasstr===s)){const t={tid:e,parasstr:s};n>0&&(t.idx=n),r&&(t.sapFlag=!0),this.#c.subtasks.push(t)}}}return!0}async#ht(t){this.#kt(t);const e=this.#ft(t,"subtype"),r=this.#ft(t,"errname"),a=this.#ft(t,"wait");if("sleep"===e){const e=parseInt(this.#ft(t,"minms")),r=parseInt(this.#ft(t,"maxms")),s=(0,import_utils9.getRandomInt)(e,r);return await(0,import_utils9.sleep)(s),await this.#Ot(a),!0}const s=this.#c.page;if(!s)return this.#mt("cfginvalid","browserPageNeeded"),!1;if("element"===e){const e=this.#St(t,"element",!0);e.length<=0&&this.#mt("cfginvalid","actionWaitElement");const a=this.#ft(e[0],"loc"),i=parseInt(this.#ft(t,"timeout")),n=this.#ft(t,"state");try{await s.waitForElement(a,{timeout:i,state:n})}catch(t){logerr(t),this.#mt(r,"actionWaitElement")}}else if("navigation"===e){const e=parseInt(this.#ft(t,"timeout")),a=this.#ft(t,"waituntil"),i=this.#ft(t,"url");a||this.#mt("cfginvalid","actionWaitNavigation");try{if(i){const t=new RegExp(i);await s.waitForNavigation({url:t,timeout:e,waitUntil:a})}else await s.waitForNavigation({timeout:e,waitUntil:a})}catch(t){logerr(t),this.#mt(r,"actionWaitNavigation")}}else this.#mt("cfginvalid","actionWait");return await this.#Ot(a),!0}async#ee(t){if(!Array.isArray(t))throw new Error("##te  Ivalid action configs(actions)");for(const e of t)try{const t=this.#ft(e,"type");if(this.#F.includes(t)||this.#F.includes("operate")&&appConfig.OPERATE_ACTIONS.includes(t))continue;if(await this.#B[t].call(this,e),this.#c.continueFlag||this.#c.breakFlag)break}catch(t){throw this.#c.credits>=0&&(loginfo(`##te  tId${this.#l}, aCfg: ${JSON.stringify(e).slice(0,200)}`),loginfo(`##te  err.stack: ${t.stack}`)),t}}async#oe(){try{const t=this.#c.page;t&&this.#$.length>0&&await injectScripts(t,this.#$);const e=this.#St(this.#P,"action");await this.#ee(e),this.emit(appConfig.EVENT_TYPES.taskComplete,{code:0,message:"ok",data:null})}catch(t){this.emit(appConfig.EVENT_TYPES.taskComplete,{code:1001,message:"failed",data:t})}}#ce(){const t={};for(const[e,r]of Object.entries(this.#q))"string"==typeof e&&"string"==typeof r&&(t[e]=r);return t}#le(){const t=this.#c.page;if(t){t.on(appConfig.EVENT_TYPES.pageClose,()=>{this.emit(appConfig.EVENT_TYPES.taskComplete,{code:1005,message:"pageClosed",data:null})});const e=this.#D.maxExecutionDuration;e>0&&setTimeout(()=>this.emit(appConfig.EVENT_TYPES.taskComplete,{code:1009,message:"timeout",data:null}),1e3*e),t.pageInfo().relatedId<=0&&t.on(appConfig.EVENT_TYPES.pagePopup,async t=>{loginfo("LsdPage received popup event");const e=t;await this.processLoginAndCaptchaPage(e,"popup")})}}#pe(){const t=this.#c.page;t&&(t.removeAllListeners(appConfig.EVENT_TYPES.pagePopup),t.removeAllListeners(appConfig.EVENT_TYPES.pageClose)),this.removeAllListeners(appConfig.EVENT_TYPES.taskComplete)}async run(){const t={templateId:this.#l,parasStr:this.#U,credits:0,execData:{},subtasks:[]},e=this.#c.page;if(e){const t=this.#Et();loginfo(`##tmp  --  start to run tId${this.#l} kId${this.#p} bIdx${t.browserIdx} pIdx${t.pageIdx}`)}else loginfo(`##tmp  --  start to run tId${this.#l} kId${this.#p} bIdx0, pIdx0`);let r;try{this.#le(),this.#oe();const[a]=await(0,import_node_events2.once)(this,appConfig.EVENT_TYPES.taskComplete);if(0===a.code);else if(1001===a.code){if(a.data instanceof Error)throw a.data;logerr(a.data)}else 1005===a.code?this.#mt("other","pageClosed"):1009===a.code&&this.#mt("tetimeout","teTimeout");e&&this.#W!==e.url()&&loginfo(`##te  kId${this.#p} endUrl: ${e.url()}`);const s=this.#c.subtasks.length;s>0&&loginfo(`##te  kId${this.#p} has ${s} subtasks`);for(let t of Object.keys(this.#c.execData)){const e=this.#c.execData[t].length;loginfo(`##te  kId${this.#p} tablename ${t} has ${e}`)}return this.#pe(),loginfo(`##tmp  teSucceeded kId${this.#p} tId${this.#l}`),t.execData=this.#c.execData,t.subtasks=this.#c.subtasks,t.credits=this.#c.credits,this.#c.subtaskResults.length>0&&(r=this.#c.subtaskResults.map(t=>t.taskData)),await this.__whenTaskEndFun(),t.credits<-30&&logwarn(`##te taskId ${this.#p} credits ${t.credits}`),{taskData:t,subtaskDatas:r,newStateData:this.#c.newStateData?this.#c.newStateData:void 0}}catch(a){t.execData=this.#c.execData,t.subtasks=this.#c.subtasks,t.credits=this.#c.credits,this.#c.subtaskResults.length>0&&(r=this.#c.subtaskResults.map(t=>t.taskData));const s=this.#c.newStateData?this.#c.newStateData:void 0,i="string"==typeof a?.stack?a.stack:"unkown",n={taskId:this.#p,message:"unknown",stack:i,variables:this.#ce()};if(this.#pe(),await this.__whenTaskEndFun(),this.#c.credits>=0)logerr(`##te  Exeception in ScraperTask.run: kId ${this.#p}`),logerr(a);else{if(-1===this.#c.credits)return{taskData:t,subtaskDatas:r,newStateData:s,misc:n};if(this.#c.credits>=-30)return{taskData:t,subtaskDatas:r,newStateData:s,misc:n};if(2===this.#p&&"check"===this.#U&&e){logdbg(`##cap  loginCheckFailed capabilityId ${0} credits: ${this.#c.credits}`)}else 11===this.#p||-69===this.#c.credits||logwarn(`##te  taskExec run execption: templateId ${this.#l}, paras ${this.#U}, credits ${this.#c.credits}`)}return logwarn(`##tmp  taskExec run execption: templateId ${this.#l}, paras ${this.#U}, credits ${this.#c.credits}`),t.credits=this.#c.credits>=0?-99:this.#c.credits,{taskData:t,subtaskDatas:r,newStateData:s,misc:n}}}async __performMiscAction(t){this.#kt(t);const e=this.#St(t,"misc");if(e.length<=0)return logerr(`##templ ${this.#l} invalid template ${this.#l} in performMiscAction`),!1;for(const t of e){const e=this.#ft(t,"type");switch(e){case"delsyncdbdata":case"extractdata":this.#mt("cfginvlid","notSupported");break;case"getstatedata":await this.#ie(t);break;default:(0,import_utils9.unreachable)(e)}}return!0}async __processDbquery(t,e){return(t||e)&&this.#mt("cfginvlid","notSupported"),""}async __processBrowserOcrCfg(t,e){return(t||e)&&this.#mt("cfginvlid","notSupported"),""}async __decodeFontInContent(t,e,r){return(t||e||r)&&this.#mt("cfginvlid","notSupported"),""}async processLoginAndCaptchaPage(t,e){return!t&&!e}async __whenTaskEndFun(){return!0}},path5=__toESM(require("path"),1),fsp2=__toESM(require("fs/promises"),1),import_utils10=require("@letsscrapedata/utils");async function moveDataFiles(){try{const t=appConfig.cfg.exportsDir;if(!t||!(0,import_utils10.isDirectory)(t))return!1;const e=await(0,import_utils10.filesInDir)(t,"general","dat_");if(0===e.length)return!0;const r=(0,import_utils10.getLocalDateNumber)(),a=path5.join(t,String(r));await(0,import_utils10.fsCheckOrCreateDir)(a,!0,!0);for(const r of e){const e=path5.join(t,r),s=path5.join(a,r);await fsp2.rename(e,s)}return!0}catch(t){return logerr(t),!1}}async function writeDataIntoOneFile(t,e,r){if(e.length<=0)return!1;const a=appConfig.cfg.dataFileFormat;if("jsonl"!==a&&!await(0,import_utils10.isFile)(t)){const a=Array.from(Object.keys(e[0]));await fsp2.writeFile(t,a.join(r).concat("\n"))}const s="jsonl"===a?e.map(t=>JSON.stringify(t)).join("\n"):e.map(t=>Object.values(t).join(r)).join("\n");return await fsp2.appendFile(t,s.concat("\n")),!0}async function saveExecDataOfTask(t){try{const{exportsDir:e,dataFileFormat:r,columnSeperator:a}=appConfig.cfg;if(!(e&&(0,import_utils10.isDirectory)(e)&&r&&a))return!1;for(const[s,i]of Object.entries(t)){const t=path5.join(e,`${s}.${r}`);await writeDataIntoOneFile(t,i,a)}return!0}catch(t){return logerr(t),!1}}async function performOneTask(t,e,r,a="indAsync",s="",i=0,n=appConfig.cfg.useNickName){try{const o=await TemplateManagerInScraper.getTemplateConfig(t,s),{template:c,actionConfigs:l,fontsConfig:p,paraCfgs:g,attrsInXml:h,captchaTypes:d,datatableMap:f}=o,u={template:c,actionConfigs:l,fontsConfig:p},{skipActions:m="",extractColumns:y="",defaultParasStr:C}=h||{},w=e||(appConfig.cfg.useParasstrInXmlIfNeeded&&C?C:""),k=TaskParser.getInParas(w,g),_=1===i&&h?{skipActions:m?m.split(","):[],extractColumns:y?y.split(","):[]}:{};_._captchaOptions={captchaTypes:d};const v=new TaskScraper(u,k,r,_),S=await v.run();if(n&&(S.taskData.execData=TaskParser.convertExecData(S.taskData.execData,f)),"indSync"===a||"indAsync"===a)if(S.taskData.credits>=-1)loginfo(`Task are completed: templateId ${t}, parasStr ${e}`),await saveExecDataOfTask(S.taskData.execData);else if("indSync"===a){const r={dat_0000000000000001:[{c01:String(t),c02:e,c03:String(S.taskData.credits),c04:(0,import_utils11.getLocalDateString)()}]};await saveExecDataOfTask(r)}return S}catch(r){const s={taskId:0,message:r.message,stack:r.stack,variables:{}};if("indSync"===a){const r={dat_0000000000000001:[{c01:String(t),c02:e,c03:"-1",c04:(0,import_utils11.getLocalDateString)()}]};await saveExecDataOfTask(r)}return{taskData:{templateId:t,parasStr:e,credits:0,execData:{},subtasks:[]},subtaskDatas:[],misc:s}}}appConfig.performOneTask=performOneTask;var path7=__toESM(require("path"),1),import_utils12=require("@letsscrapedata/utils"),import_controller3=require("@letsscrapedata/controller"),BrowserManagerInScraper=class{#ge;#he;#de;#fe;#ue(t){const e=this.#ge.templateParas.find(e=>e.templateId===t);return e?.maxConncurrency?e.maxConncurrency:this.#ge.maxConcurrency}async#me(t,e){const r=await import_controller3.controller.newApiContext({proxy:e}),a="playwright"===t.browser().browserControllerType()?t.apiContext():null;this.#fe.push({proxy:e,browserContext:t,browserApiContext:a,standaloneApiContext:r,templateMap:new Map})}constructor(){const{useStealthPlugin:t,browserControllerType:e,browserType:r,lsdLaunchOptions:a,lsdConnectOptions:s,maxConcurrency:i,templateParas:n}=appConfig.cfg,o=Array.isArray(appConfig.cfg.browserConfigs)&&appConfig.cfg.browserConfigs.length>0?appConfig.cfg.browserConfigs:[{browserControllerType:e,browserType:r,proxyUrl:""}];this.#ge={useStealthPlugin:t,browserControllerType:e,browserType:r,lsdLaunchOptions:a,lsdConnectOptions:s,browserConfigs:o,maxConcurrency:i,templateParas:n},this.#he=import_controller3.controller,["playwright","puppeteer"].includes(e),this.#de=[],this.#fe=[]}async init(){const{browserControllerType:t,browserType:e,lsdLaunchOptions:r,lsdConnectOptions:a}=this.#ge;for(const s of this.#ge.browserConfigs){const{browserControllerType:i=t,browserType:n=e,browserUrl:o,proxyUrl:c=""}=s,l=this.#ye(c);if(o){const t=Object.assign({},a,{browserUrl:o}),e=await this.#he.connect(i,n,t);this.#de.push(e);for(const t of e.browserContexts())await this.#me(t,l)}else if("string"==typeof c){const t=c?l:void 0,e=Object.assign({},r,{proxy:t}),a=await this.#he.launch(i,n,e);this.#de.push(a);for(const t of a.browserContexts())await this.#me(t,l)}}return await(0,import_utils12.sleep)(2e3),!0}#ye(t,e="",r="",a=""){return{proxyUrl:t,proxyId:e||t,username:r,password:a}}async getTaskNetworkContext(t=0){if(0===this.#fe.length)throw new Error("please call init() first to launch or connect browsers.");let e;const r=(0,import_utils12.getRandomArrayIdx)(this.#fe.length);for(const a of r){const r=this.#fe[a],{proxy:s,browserContext:i,browserApiContext:n,standaloneApiContext:o,templateMap:c}=r;if(t>0&&(e=c.get(t),e||(e={templateId:t,maxConcurrency:this.#ue(t),running:0},c.set(t,e)),e.maxConcurrency<=e.running))continue;if(!i.hasFreePage())continue;const l=await i.getPage();return l?(t>0&&e&&e.running++,l.setPageInfo({relatedId:t}),{proxy:s,page:l,browserApiContext:n,standaloneApiContext:o}):null}return null}async freePage(t){const e=t.pageInfo().relatedId;if(e>0){const r=t.browserContext().id(),a=this.#fe.find(t=>t.browserContext.id()===r);if(!a)throw new Error("No browserContext has the page to free.");const s=a.templateMap.get(e);if(!s)throw new Error(`No templateMapInfo has templateId ${e} in ${r}.`);s.running--}return await t.free(),!0}},fsp3=__toESM(require("fs/promises"),1),path6=__toESM(require("path"),1),import_utils13=require("@letsscrapedata/utils"),SchedulerInScraper=class t{#Ce;#we;#ke;#_e;runningTaskSet;static#ve="\n";static#Se=!1;constructor(t){this.#Ce=new Map,this.#we=appConfig.cfg.taskDir,this.#ke=appConfig.cfg.exitWhenCompleted,this.#_e=t,this.runningTaskSet=new Set}async#be(e){try{const r=await fsp3.readFile(e,"utf8");return r.split(t.#ve)}catch(t){return[]}}static#xe=new Map;async#Ee(e,r){if(t.#xe.get(r)){for(let e=0;e<50&&(await(0,import_utils13.sleep)(200),t.#xe.get(r));e++);if(t.#xe.get(r))return!1}t.#xe.set(r,!0);try{const a=path6.join(this.#we,`templ${r}.txt`);return e.length>0?await fsp3.writeFile(a,e.join(t.#ve)):await(0,import_utils13.isFile)(a)&&await fsp3.unlink(a),t.#xe.delete(r),!0}catch(e){return logwarn(`Failed to write task queue of tempatedId ${r} into file`),t.#xe.delete(r),!1}}async#Te(){const t=appConfig.cfg.taskDir;if(!t||!(0,import_utils13.isDirectory)(t))return!1;const e=await(0,import_utils13.filesInDir)(t,"general","templ","txt");for(const r of e){const e=parseInt(r.slice(5,-4)),a=await this.#be(path6.join(t,r));if(a.length>0){loginfo(`load unfinished ${a.length} tasks of templateId ${e}`);const t=this.#Ce.get(e);t?a.forEach(e=>t.push(e)):this.#Ce.set(e,a)}}return!0}async addNewTask(t,e){const{tid:r,parasstrs:a}=t;if(r<=0||0===a.length)return!1;let s=this.#Ce.get(r);return s||(s=[],this.#Ce.set(r,s)),a.forEach(t=>{s.includes(t)||s.push(t)}),await this.#Ee(s,r),e&&await this.tryToExecuteTasks([r]),!0}async init(t=[]){appConfig.cfg.moveDataWhenStart&&await moveDataFiles(),appConfig.cfg.loadUnfinishedTasks&&await this.#Te();for(const e of t){const{tid:t,parasstrs:r}=e;this.addNewTask({tid:t,parasstrs:r},!1)}return await this.tryToExecuteTasks(),setInterval(this.tryToExecuteTasks.bind(this),6e4),!0}#Ae(t,e){return`${t}@@${e}`}async#Ie(t,e,r){const a=await performOneTask(t,e,r,"indAsync","",0,!0),{credits:s,subtasks:i}=a.taskData;let n=new Set;const o=this.#Ce.get(t);if(o){const r=o.findIndex(t=>t===e);if(r<0?logerr(`Task queue of templateId ${t} has no parasStr: ${e}`):o.splice(r,1),s>=-1){if(0===o.length&&this.#Ce.delete(t),i.length>0){n=new Set(i.map(t=>t.tid));for(const t of n){const e=i.filter(e=>e.tid===t).map(t=>t.parasstr);await this.addNewTask({tid:t,parasstrs:e},!1)}}await this.#Ee(o,t)}else o.push(e),await this.#Ee(o,t);this.runningTaskSet.delete(this.#Ae(t,e))}else logerr(`No task queue of templateId ${t}`);return r.page&&await this.#_e.freePage(r.page),appConfig.cfg.minMiliseconds>0&&await(0,import_utils13.sleep)(appConfig.cfg.minMiliseconds),await this.tryToExecuteTasks([t],1),n.size>0&&await this.tryToExecuteTasks(),!0}async tryToExecuteTasks(e=[],r=0){if(loginfo(`----    tryToExecuteTasks templateIds ${e.join(",")} : `,this.#Ce),t.#Se)return logdbg("----    another tryToExecuteTasks is running now"),!0;this.#ke&&0===this.#Ce.size&&process.exit(0),t.#Se=!0;try{let a=0;0===e.length&&(e=Array.from(this.#Ce.keys())),logdbg(`templateIds: ${e.join(",")}`);for(const s of e){const e=this.#Ce.get(s);if(!e)break;const i=e.length;for(let n=0;n<i;n++){const i=e[n],o=this.#Ae(s,i);if(this.runningTaskSet.has(o))continue;const c=await this.#_e.getTaskNetworkContext(s);if(!c){logdbg(`No page to execute ${o}`);break}if(this.runningTaskSet.add(o),this.#Ie(s,i,c),a++,r>0&&a>=r)return t.#Se=!1,!0}}return t.#Se=!1,!0}catch(e){return logdbg(e),t.#Se=!1,!1}}},import_utils14=require("@letsscrapedata/utils");async function updateScraperConfig(t){const e=appConfig.cfg,{baseDir:r,inputFileDirPart:a,useStealthPlugin:s,browserControllerType:i,browserType:n,lsdLaunchOptions:o,lsdConnectOptions:c,browserConfigs:l,urlPrefix:p,maxConcurrency:g,readCode:h,templateParas:d,totalMaxConcurrency:f,minMiliseconds:u,exitWhenCompleted:m,useParasstrInXmlIfNeeded:y,loadUnfinishedTasks:C,moveDataWhenStart:w,dataFileFormat:k,useNickName:_,columnSeperator:v}=t;if(r)e.baseDir=r;else{const t=process.cwd();e.baseDir=path7.join(t,"scraperdata")}if(await(0,import_utils14.fsCheckOrCreateDir)(e.baseDir,!0,!0),e.baseDownloadDir=path7.join(e.baseDir,"downloads"),await(0,import_utils14.fsCheckOrCreateDir)(e.baseDownloadDir,!0,!0),e.baseFileDir=path7.join(e.baseDir,"files"),await(0,import_utils14.fsCheckOrCreateDir)(e.baseFileDir,!0,!0),t.templateDir?e.templateDir=t.templateDir:e.templateDir=path7.join(e.baseDir,"template"),await(0,import_utils14.fsCheckOrCreateDir)(e.templateDir,!0,!0),e.taskDir=path7.join(e.baseDir,"task"),await(0,import_utils14.fsCheckOrCreateDir)(e.taskDir,!0,!0),e.exportsDir=path7.join(e.baseDir,"exports"),await(0,import_utils14.fsCheckOrCreateDir)(e.exportsDir,!0,!0),a&&(e.inputFileDirPart=a),i&&(e.browserControllerType=i),n&&(e.browserType=n),o&&(e.lsdLaunchOptions=o),c&&(e.lsdConnectOptions=c),l&&(e.browserConfigs=l),p&&(e.urlPrefix=p),g&&(e.maxConcurrency=g),h&&(e.readCode=h),d&&(e.templateParas=d),f&&(e.totalMaxConcurrency=f),"number"==typeof u&&(e.minMiliseconds=u),"boolean"==typeof s&&(e.useStealthPlugin=s),"boolean"==typeof m&&(e.exitWhenCompleted=m),"boolean"==typeof y&&(e.useParasstrInXmlIfNeeded=y),"boolean"==typeof C&&(e.loadUnfinishedTasks=C),"boolean"==typeof w&&(e.moveDataWhenStart=w),k&&(e.dataFileFormat=k),"boolean"==typeof _&&(e.useNickName=_),"csv"===e.dataFileFormat?e.columnSeperator=",":"tsv"===e.dataFileFormat?e.columnSeperator="\t":"txt"===e.dataFileFormat&&v&&(e.columnSeperator=v),t.captcha){const{clientKey:r}=t.captcha;"string"==typeof r&&r&&(e.captcha.clientKey=r)}return!0}async function scraper(t=[],e={}){await updateScraperConfig(e);const r=new BrowserManagerInScraper;await r.init();const a=new SchedulerInScraper(r);return await a.init(t),!0}
+"use strict";var __create=Object.create,__defProp=Object.defineProperty,__getOwnPropDesc=Object.getOwnPropertyDescriptor,__getOwnPropNames=Object.getOwnPropertyNames,__getProtoOf=Object.getPrototypeOf,__hasOwnProp=Object.prototype.hasOwnProperty,__export=(t,e)=>{for(var r in e)__defProp(t,r,{get:e[r],enumerable:!0})},__copyProps=(t,e,r,a)=>{if(e&&"object"==typeof e||"function"==typeof e)for(let s of __getOwnPropNames(e))__hasOwnProp.call(t,s)||s===r||__defProp(t,s,{get:()=>e[s],enumerable:!(a=__getOwnPropDesc(e,s))||a.enumerable});return t},__toESM=(t,e,r)=>(r=null!=t?__create(__getProtoOf(t)):{},__copyProps(!e&&t&&t.__esModule?r:__defProp(r,"default",{value:t,enumerable:!0}),t)),__toCommonJS=t=>__copyProps(__defProp({},"__esModule",{value:!0}),t),src_exports={};__export(src_exports,{TaskParser:()=>TaskParser,TemplateManagerInScraper:()=>TemplateManagerInScraper,performOneTask:()=>performOneTask,scraper:()=>scraper,setScraperLogFun:()=>setScraperLogFun,updateScraperConfig:()=>updateScraperConfig}),module.exports=__toCommonJS(src_exports);var import_controller=require("@letsscrapedata/controller"),import_proxy=require("@letsscrapedata/proxy"),import_utils=require("@letsscrapedata/utils"),pkgLog=import_utils.log;function setScraperLogFun(t){return"function"==typeof t&&(pkgLog=t,(0,import_controller.setControllerLogFun)(pkgLog),(0,import_proxy.setProxyLogFun)(pkgLog),!0)}async function logdbg(...t){await pkgLog(import_utils.LogLevel.DBG,...t)}async function loginfo(...t){await pkgLog(import_utils.LogLevel.INF,...t)}async function logwarn(...t){await pkgLog(import_utils.LogLevel.WRN,...t)}async function logerr(...t){await pkgLog(import_utils.LogLevel.ERR,...t)}(0,import_controller.setControllerLogFun)(pkgLog),(0,import_proxy.setProxyLogFun)(pkgLog),(0,import_utils.setLogLevel)(import_utils.LogLevel.INF);var import_xml2js_lsd=__toESM(require("xml2js-lsd"),1),myfunCfg={type:void 0,arg2:"",arg3:"",arg4:"",arg5:""},defaultSplitOfParases="[\r\n\t,]+",defaultSeperatorOfParas="::",newLineChar="\n",defaultElementCfg={template:{tid:void 0,version:void 0,commited:!1},attrs:{},attr:{name:void 0,value:void 0},paras:{},para:{paraname:void 0,name:"",desc:"",uitype:"",defval:"",listid:"0",listparas:"",min:"",max:"",pattern:"",alert:""},element:{loc:void 0,source:"default",absolute:!1},iframe:{srcprefix:"",loc:""},templstr:{templ:void 0},elecontent_attr:{type:void 0,attrname:void 0,attrname2:"",attrname3:"",attrname4:"",trim:!1,pattern:"",boolattr:!1,multieles:!0,join:newLineChar,line:!1,absolute:!1},elecontent_imgbase64:{type:void 0},elecontent_innerhtml:{type:void 0},elecontent_length:{type:void 0},elecontent_outerhtml:{type:void 0},elecontent_text:{type:void 0,trim:!0,multieles:!0,join:newLineChar,line:!1,top:!1},decodefontsvg:{outtype:"text"},decodefontttf:{fflocs:"",intype:"outerhtml",outtype:"text"},ocrcfg_api:{type:void 0,nlchars:"",seperator:!1,lang:"zh"},ocrcfg_tesseract:{type:void 0,nlchars:"",seperator:!1,lang:"zh"},actions:{},action_break:{type:void 0,id:"0"},action_captcha:{type:void 0,try:"1",errname:"captchafailed",id:"0"},action_click:{type:void 0,wait:"-1",gen:!0,cap:!0,clicktype:"click",try:"1",errname:"cfginvalid",datapage:"current",popupsubtask:!1,login:!0,captcha:!0,navigate:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",downloadpath:"",filename:"",pathvarname:"",id:"0"},action_continue:{type:void 0,id:"0"},action_misc:{type:void 0,id:"0"},misc_delsyncdbdata:{type:void 0,tids:"",retention:"2592000",minretention:"86400"},misc_extractdata:{type:void 0,name:void 0,method:"put",paras:!0,execdata:!0,subtasks:!0,credits:!0},misc_getstatedata:{type:void 0,cookies:"new",localstorage:"new",headers:"none",userdata:"none"},action_exit:{type:void 0,errname:void 0,id:"0"},action_extract:{type:void 0,tabname:void 0,id:"0"},column_element:{type:void 0,colname:void 0,nickname:"",completed:!1,setvar:!1,datatype:"string",errname:"ignore",valerrname:"ignore",pattern:"",flags:"",id:"0",discarded:!1},column_ocr:{type:void 0,colname:void 0,nickname:void 0,completed:!1,setvar:!1,datatype:"string",valerrname:"ignore",pattern:"",flags:"",id:"0",discarded:!1},column_subtask:{type:void 0,colname:void 0,nickname:void 0,completed:!1,setvar:!1,datatype:"string",valerrname:"ignore",pattern:"",flags:"",id:"0",discarded:!1},column_templstr:{type:void 0,colname:void 0,nickname:void 0,completed:!1,setvar:!1,datatype:"string",valerrname:"ignore",pattern:"",flags:"",id:"0",discarded:!1},action_goto:{type:void 0,url:void 0,reuse:!1,wait:"-1",encodeuri:!1,gen:!0,cap:!0,datapage:"current",popupsubtask:!1,login:!0,captcha:!0,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",headers:"",referer:"",id:"0"},action_hover:{type:void 0,try:"1",gen:!1,cap:!1,wait:"-1",errname:"cfginvalid",id:"0"},action_ifelse:{type:void 0,id:"0"},condition_element:{type:void 0},condition_else:{type:void 0},condition_templstr:{type:void 0},action_input:{type:void 0,content:void 0,enter:!1,replace:!1,gen:!1,cap:!1,try:"1",wait:"-1",errname:"cfginvalid",datapage:"current",popupsubtask:!1,login:!1,captcha:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",id:"0"},action_intercept_clear:{type:void 0,subtype:void 0},action_intercept_set:{type:void 0,subtype:void 0},request_abort:{type:void 0,method:"all",url:"",resourcetype:"image",postdata:""},response_cache:{type:void 0,method:"all",url:"",resourcetype:"xhr",postdata:"",minsize:"0",maxsize:"0",contenttype:"text",responses:!0,tabname:"",requestheaders:!1,responseheaders:!1},response_save:{type:void 0,method:"all",url:"",resourcetype:"image",postdata:"",minsize:"0",maxsize:"0",basedir:"response",pathtype:"path_name",hashmethod:"MD5",hostnameparts:"0",extname:"",validextnames:"",params:"",encoding:"",tabname:""},action_loopdowhile_element:{type:void 0,subtype:void 0,iswhile:!1,varname:"",maxloops:"100",click:!1,navigate:!0,gen:!0,cap:!0,errname:"cfginvalid",wait:"-1",id:"0"},action_loopdowhile_templstr:{type:void 0,subtype:void 0,iswhile:!1,varname:"",maxloops:"100",id:"0"},action_loopfor:{type:void 0,from:void 0,to:void 0,step:"1",roundtype:"ceil",varname:void 0,maxloops:"0",errname:"ignore",id:"0"},action_loopineles:{type:void 0,varname:"",maxloops:"0",start:"1",end:"0",step:"1",errname:"ignore",id:"0"},action_loopinstr:{type:void 0,list:void 0,split:defaultSplitOfParases,varname:void 0,maxloops:"0",trim:!0,errname:"ignore",id:"0"},action_scroll_by:{type:void 0,subtype:void 0,height:"100",unit:"windowpct",maxtimes:"1",interval:"800",gen:!1,cap:!1,datapage:"current",popupsubtask:!1,login:!1,captcha:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",id:"0"},action_scroll_intoview:{type:void 0,subtype:void 0,gen:!1,cap:!1,errname:"cfginvalid",datapage:"current",popupsubtask:!1,login:!1,captcha:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",id:"0"},action_scroll_to:{type:void 0,subtype:void 0,height:"100",unit:"bodypct",gen:!1,cap:!1,datapage:"current",popupsubtask:!1,login:!1,captcha:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",id:"0"},action_select:{type:void 0,selecttype:"value",selectval:"",gen:!1,cap:!1,try:"1",wait:"-1",errname:"cfginvalid",datapage:"current",popupsubtask:!1,login:!1,captcha:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",id:"0"},dbquery:{dbname:void 0,sql:void 0,sqlparas:"",parasplit:defaultSeperatorOfParas,join:newLineChar},action_setvar_dbquery:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},action_setvar_element:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",try:"1",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},file:{path:void 0,encoding:"utf8"},action_setvar_file:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",proxy:!0,valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},action_setvar_get:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},action_setvar_ocr:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},action_setvar_subtask:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},action_setvar_templstr:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},subtask:{tid:void 0,parasstr:void 0,idx:"",minlen:"1",errname:"cfginvalid",popuppage:!1},action_subtask:{type:void 0,subtasks:"",sameasparent:!1,id:"0"},action_wait_element:{type:void 0,subtype:void 0,timeout:"30000",state:"visible",errname:"ignore",wait:"-1",id:"0"},action_wait_navigation:{type:void 0,subtype:void 0,timeout:"30000",waituntil:"load",url:"",errname:"cfginvalid",wait:"-1",id:"0"},action_wait_sleep:{type:void 0,subtype:void 0,minms:"5000",maxms:"10000",errname:"ignore",wait:"-1",id:"0"},fonts:{},fontsvg:{exloc:"head link[rel='stylesheet']",inloc:"",csmaptype:"general",bsfilter:""},fontttf:{exloc:"head link[rel='stylesheet']",inloc:"",minuc:"57344",maxuc:"63744",startidx:"-1",fsfilter:"",fufilter:"",parsetype:""},fontselector:{name:void 0,fontfamily:void 0},fontfamily:{name:void 0,fontcodes:void 0,fontchars:void 0},fontcodes:{name:void 0,codes:void 0},fontchars:{name:void 0,chars:void 0},myfuns:{},myfun:{name:void 0,arg2:"",arg3:"",arg4:"",arg5:"",funbody:void 0},captcha_amazon:{type:void 0},captcha_funcaptcha:{type:void 0},captcha_geetest:{type:void 0},captcha_keycaptcha:{type:void 0},captcha_mtcaptcha:{type:void 0},captcha_recaptcha:{type:void 0,minscore:"0.3"},captcha_turnstile:{type:void 0},image_element:{type:void 0},comment_element:{type:void 0,attr:"textContent"},input_element:{type:void 0,enter:!1},submit_element:{type:void 0},check_result:{type:void 0,attr:"",failedstr:""},captcha_text:{type:void 0,case:!1,lang:"en",math:!1,maxlen:"0",minlen:"0",question:!1,space:!1,texttype:"any"},captcha_coordinate:{type:void 0,lang:"en",maxclicks:"0",minclicks:"1"},transform:{defaultval:"",usevar:!1},fun_c_attr:{type:void 0,subtype:void 0,loc:void 0,idx:"0",attrname:void 0},fun_c_data:{type:void 0,subtype:void 0,loc:void 0,idx:"0",attrname:void 0},fun_c_html:{type:void 0,subtype:void 0,loc:void 0,idx:"0"},fun_c_length:{type:void 0,subtype:void 0,loc:void 0},fun_c_text:{type:void 0,subtype:void 0,loc:void 0,idx:"-9999"},fun_ceil:{type:void 0},fun_closingsubstr:{type:void 0,openchar:void 0,startstr:"",position:"0"},fun_compress:{type:void 0,method:"gzip",flush:"",finishflush:"",chunksize:"0",windowbits:"0",level:"",memlevel:"",strategy:"",sourceencoding:"utf8",targetencoding:"base64"},fun_concat:{type:void 0,str1:void 0,str2:"",str3:"",str4:"",str5:""},fun_convertencoding:{type:void 0,sourceencoding:"base64",targetencoding:"utf8",start:"0",end:"0"},fun_decompress:{type:void 0,method:"gunzip",flush:"",finishflush:"",chunksize:"0",windowbits:"0",sourceencoding:"base64",targetencoding:"utf8",start:"0",end:"0"},fun_decode:{type:void 0,str1:"",val1:"",str2:"",val2:"",str3:"",val3:"",str4:"",val4:"",str5:"",val5:"",defaultval:""},fun_floor:{type:void 0},fun_hashcode:{type:void 0,hashmethod:"MD5"},fun_includes:{type:void 0,searchstr:void 0},fun_insert:{type:void 0,str:void 0},fun_itemstoobj:{type:void 0,split:defaultSplitOfParases,kvsplit:":",keytrim:!0,keynows:!1,valuetrim:!0,valuenows:!1},fun_jsonparse:{type:void 0,key1:"",key2:"",key3:"",key4:"",key5:"",key6:"",stringify:!1,keys:!1,length:!1,srctype:"json"},fun_length:{type:void 0},fun_matchall:{type:void 0,pattern:void 0,flags:"g",itemvalue:"match[0]",filter:"",start:"0",end:"0",resulttype:"all",join:newLineChar},fun_max:{type:void 0,split:defaultSplitOfParases},fun_min:{type:void 0,split:defaultSplitOfParases},fun_numeq:{type:void 0,num:void 0},fun_numge:{type:void 0,num:void 0},fun_numgt:{type:void 0,num:void 0},fun_numle:{type:void 0,num:void 0},fun_numlt:{type:void 0,num:void 0},fun_notincludes:{type:void 0,searchstr:void 0},fun_padend:{type:void 0,targetlen:void 0,padstr:" "},fun_padstart:{type:void 0,targetlen:void 0,padstr:" "},fun_parsefloat:{type:void 0},fun_parseint:{type:void 0},fun_randomarryidxes:{type:void 0,join:newLineChar},fun_replace:{type:void 0,substr:void 0,newstr:void 0,regexp:!1,flags:""},fun_round:{type:void 0},fun_setcomplement:{type:void 0,array2:void 0,split1:defaultSplitOfParases,split2:defaultSplitOfParases,sorttype:"nosort",valtype:"number",limit:"0",join:newLineChar},fun_setdifference:{type:void 0,array2:void 0,split1:defaultSplitOfParases,split2:defaultSplitOfParases,sorttype:"nosort",valtype:"number",limit:"0",join:newLineChar},fun_setintersection:{type:void 0,array2:void 0,split1:defaultSplitOfParases,split2:defaultSplitOfParases,sorttype:"nosort",valtype:"number",limit:"0",join:newLineChar},fun_setunion:{type:void 0,array2:void 0,split1:defaultSplitOfParases,split2:defaultSplitOfParases,sorttype:"nosort",valtype:"number",limit:"0",join:newLineChar},fun_slice:{type:void 0,beginidx:void 0,endidx:""},fun_shuffle:{type:void 0,split:defaultSplitOfParases},fun_sort:{type:void 0,split:defaultSplitOfParases,valtype:"number",sorttype:"asc",join:newLineChar},fun_specialconvert:{type:void 0,method:void 0,para1:"",para2:"",para3:"",para4:"",para5:""},fun_split:{type:void 0,split:defaultSplitOfParases,limit:"0",filter:"",start:"0",end:"0",resulttype:"all",val:"",insertstr:"",join:newLineChar},fun_streq:{type:void 0,str:void 0},fun_substrafter:{type:void 0,substr:void 0,position:"0",last:!1,include:!1},fun_substrbefore:{type:void 0,substr:void 0,position:"0",last:!1,include:!1},fun_tobool:{type:void 0},fun_tolowercase:{type:void 0},fun_tonum:{type:void 0},fun_touppercase:{type:void 0},fun_trim:{type:void 0},fun_trimend:{type:void 0},fun_trimstart:{type:void 0},fun_uniq:{type:void 0,split:defaultSplitOfParases,join:newLineChar},fun_urldelparams:{type:void 0,param1:void 0,param2:"",param3:""},fun_urldecode:{type:void 0,method:"URI",encoding:"utf8",percent:!0},fun_urlencode:{type:void 0,method:"URI",encoding:"utf8",percent:!0},fun_urlget:{type:void 0,attrorpath:void 0,hashmethod:"MD5",hostnameparts:"0",params:"",param:"",extname:"",validextnames:"",title:""},fun_urlsetparams:{type:void 0,param1:void 0,val1:"",param2:"",val2:"",param3:"",val3:"",replace:!0},fun_myfun1:Object.assign({},myfunCfg),fun_myfun2:Object.assign({},myfunCfg),fun_myfun3:Object.assign({},myfunCfg),fun_myfun4:Object.assign({},myfunCfg),fun_myfun5:Object.assign({},myfunCfg),get_addedtaskparas:{type:void 0,tid:void 0,limit:"10000",join:newLineChar,interval:"3600"},get_api:{type:void 0,method:"GET",url:void 0,encodeuri:!1,session:!1,headerssource:"headers",headers:"",referer:"",proxy:!0,timeout:"30000",context:"default",data:"",datatype:"json"},get_content:{type:void 0},get_cookies:{type:void 0,urls:"",domain:"",name:"",path:"",value:!1},get_datetime:{type:void 0,format:"yyyy-MM-ddThh:mm:ss"},get_execedtaskparas:{type:void 0,tid:void 0,limit:"10000",join:newLineChar,interval:"3600"},get_file:{type:void 0,url:void 0,path:"",basedir:"file",pathtype:"path_name",hashmethod:"MD5",hostnameparts:"0",params:"",extname:"",validextnames:"",pathvarname:"",proxy:!0,headers:"",referer:"",setvar:!1},httpbrowser:{name:void 0,minversion:"0",maxversion:"0",httpversion:"0"},httpdevice:{device:void 0},httpheader:{name:void 0,source:"value",value:void 0,delete:!1},httpos:{os:void 0},get_httpheaders:{type:void 0,headermethod:"genLatestWindowsChrome",origheaders:"",browserlist:"",httpversion:"0"},get_mhtml:{type:void 0,path:"",basedir:"mhtml",pathtype:"title",hashmethod:"MD5",hostnameparts:"0",pathvarname:""},get_pdf:{type:void 0,path:"",basedir:"pdf",pathtype:"title",hashmethod:"MD5",hostnameparts:"0",scale:"1",displayheaderfooter:!1,headertemplate:"",footertemplate:"",printbackground:!1,landscape:!1,pageranges:"",format:"Letter",width:"",height:"",top:"",right:"",bottom:"",left:"",screen:!1,onepage:!1,hmargin:"",setvar:!1,pathvarname:""},get_queuedcntwithparas:{type:void 0,tid:void 0,parasstr:void 0,operator:"eq"},get_queuedtaskparas:{type:void 0,tid:void 0,limit:"10000",join:newLineChar,interval:"3600"},get_random:{type:void 0,min:void 0,max:void 0},get_response:{type:void 0,requrl:"",idx:"-1",length:!1},get_screenshot:{type:void 0,basedir:"screenshot",path:"",extname:"png",quality:"-1",fullpage:!1,x:"0",y:"0",width:"0",height:"0",omitbackground:!1,setvar:!1,pathvarname:""},get_special:{type:void 0,method:void 0,parasstr:""},get_title:{type:void 0,aspath:!1},get_window:{type:void 0,key1:void 0,key2:"",key3:"",key4:"",key5:"",key6:"",keys:!1,length:!1}},appConfig={XML_ATTR_KEY:"$$",XML_CHILDREN_KEY:"children",GEN_CREDITS_BITS:16,GEN_CREDITS_AND:65535,CHECK_DNS_DOMAIN:"example.com",CHECK_POPUP_PAGE_TIMEOUT1:35e3,CHECK_POPUP_PAGE_TIMEOUT2:5e3,RESERVED_PARA_NAMES:["authInfo","inParas","sysParas"],EVENT_TYPES:{pagePopup:"pagePopup",pageClose:"pageClose",taskComplete:"taskComplete"},OPERATE_ACTIONS:["click","goto","hover","input","scroll","select","wait","captcha"],performOneTask:console.log,cfg:{exitWhenCompleted:!1,useParasstrInXmlIfNeeded:!1,loadUnfinishedTasks:!1,baseDir:"",baseDownloadDir:"",baseFileDir:"",templateDir:"",taskDir:"",exportsDir:"",inputFileDirPart:"LetsScrapeData",useStealthPlugin:!1,browserControllerType:"patchright",browserType:"chromium",lsdLaunchOptions:{headless:!1},lsdConnectOptions:{browserUrl:""},browserConfigs:[],captcha:{clientKey:""},maxConcurrency:1,readCode:"",templateParas:[],urlPrefix:"https://api.letsscrapedata.com/nologin",totalMaxConcurrency:10,minMiliseconds:2e3,moveDataWhenStart:!1,dataFileFormat:"jsonl",useNickName:!0,columnSeperator:"::"}},TaskParser=class t{static#t(e,r){try{if(!e||!r)throw new Error(`##te  Invalid paras, parentEleName ${r} in _applyDefaultElementCfg`);if("object"!=typeof e)return!0;const a=appConfig.XML_ATTR_KEY,s=appConfig.XML_CHILDREN_KEY;e[a]||(e[a]={});let i=r;e[a].type&&(i=i+"_"+e[a].type),e[a].subtype&&(i=i+"_"+e[a].subtype);const n=defaultElementCfg[i];if(!n)throw new Error(`##te  No default Cfg for eleName ${i} in _applyDefaultElementCfg`);const o=Object.assign({},n);if(Object.keys(n).forEach(t=>{let r=e[a][t];if("boolean"==typeof o[t])"true"===r||""===r?o[t]=!0:"false"===r&&(o[t]=!1);else if(r||""===r)o[t]=r;else if(void 0===o[t])throw new Error(`##te  Attribute ${t} of Element ${i} is undefined`)}),e[a]=o,!e[s])return!0;const c=Array.from(Object.keys(e[s]));for(const r of c){const a=e[s][r];1===a.length&&""===a[0]?e[s][r]=[]:a.forEach(e=>t.#t(e,r))}return!0}catch(t){throw logerr(`##te  Error parentEleName ${r} in _applyDefaultElementCfg`),new Error(t)}}static async convertXmlToJson(e,r=!0){try{const a=function(t){const e=t.indexOf("_");return e>0?t.slice(0,e):t},s={attrkey:appConfig.XML_ATTR_KEY,childkey:appConfig.XML_CHILDREN_KEY,explicitChildren:!0,tagNameProcessors:[a]},i=appConfig.XML_CHILDREN_KEY,n=await import_xml2js_lsd.default.parseStringPromise(e,s);if(!(n&&n.template&&n.template[i]&&Array.isArray(n.template[i].actions)))throw new Error(`##te  Invalid XML config string: ${e}`);return r&&t.#t(n.template,"template"),n.template}catch(t){throw logerr("##te  Exeception in convertXmlToJson"),logerr(`##te  xmlCfgStr: ${e}`),new Error(t)}}static getPartOfJsonCfg(t,e,r=!1){try{const r=t[appConfig.XML_CHILDREN_KEY][e][0];return r||null}catch(t){if(r)return null;throw logerr(`##te  Exeception when try to get ${e} in getPartOfJsonCfg`),new Error(t)}}static#e(e){try{const r=t.getPartOfJsonCfg(e,"attrs");return!Array.isArray(r?.children?.attr)||r.children.attr.length<=0?[]:r.children.attr.map(t=>t[appConfig.XML_ATTR_KEY])}catch(t){return logerr(t),[]}}static getParaCfgsFromJsonCfg(e){try{const r=t.getPartOfJsonCfg(e,"paras");return!Array.isArray(r?.children?.para)||r.children.para.length<=0?[]:r.children.para.map(t=>t[appConfig.XML_ATTR_KEY])}catch(t){return logerr(t),[]}}static getAttrsInXml(e){const r=t.#e(e),a={};return r.forEach(t=>a[t.name]=t.value),a}static#r(e,r){const a=[],s=appConfig.XML_CHILDREN_KEY;for(const i of e)if(i?.[appConfig.XML_ATTR_KEY]?.type===r)a.push(i);else if(i&&i[s]&&(i[s].action||i[s].condition)){t.#r(i[s].action?i[s].action:i[s].condition,r).forEach(t=>{a.push(t)})}return a}static getCaptchTypes(e){const r=appConfig.XML_CHILDREN_KEY,a=e[r].actions[0][r].action,s=t.#r(a,"captcha"),i=[];for(const t of s){const e=t[appConfig.XML_CHILDREN_KEY]?.captcha;Array.isArray(e)&&e.forEach(t=>{const e=t[appConfig.XML_ATTR_KEY].type;e&&i.push(e)})}return i}static getDatableMapFromJsonCfg(e){try{const r=appConfig.XML_CHILDREN_KEY,a=e[r].actions[0][r].action,s=t.#r(a,"extract"),i=new Map;for(const t of s){const e=t[appConfig.XML_ATTR_KEY].tabname;let a=i.get(e);a||(a=new Map,i.set(e,a));for(const e of t[r].column)if(!e[appConfig.XML_ATTR_KEY].discarded){const t=e[appConfig.XML_ATTR_KEY].colname,r=e[appConfig.XML_ATTR_KEY].nickname?e[appConfig.XML_ATTR_KEY].nickname:t;a.set(t,r)}}return i}catch(t){return null}}static getInParas(t,e=[],r="::"){const a={},s=t.split("@@"),i=["para","apara","rpara"];for(let t=0;t<3;t++){let n=i[t],o=e.filter(t=>t.paraname.startsWith(n)).length;if(s.length>t&&s[t].length>0){let e=s.length>t&&s[t].length>0?s[t].split(r):[];for(let t=1;t<=e.length;t++)a[`${n}${t}`]=e[t-1];for(let t=e.length+1;t<=o;t++)a[`${n}${t}`]=""}}return a}static convertExecData(t,e){if(!e)return t;const r={};for(const a of Object.keys(t)){const s=t[a],i=e.get(a);if(i&&0!==s.length){const t={};Object.keys(s[0]).forEach(e=>{const r=i.get(e);return t[e]=r||e});const e=s.map(e=>{const r={};for(const a of Object.keys(e))r[t[a]]=e[a];return r});r[a]=e}else r[a]=s}return r}},fs=__toESM(require("fs"),1),path=__toESM(require("path"),1),import_utils2=require("@letsscrapedata/utils"),TemplateManagerInScraper=class t{static#a=new Map;static#s=new Set;static#i(t){const e=appConfig.cfg.templateParas.find(e=>e.templateId===t);return e?.readCode?e.readCode:appConfig.cfg.readCode}static async#n(t,e){try{if(!e){if(!appConfig.cfg.templateDir)return null;const r=await(0,import_utils2.filesInDir)(appConfig.cfg.templateDir,"general",`templ${t}_`,"xml");let a=1===r.length?r[0]:`templ${t}.xml`;a=path.join(appConfig.cfg.templateDir,a),e=fs.readFileSync(a,"utf8")}if(!e)return null;return{templateId:t,domainId:1,defaultElementSource:"browser",maxExecutionDuration:600,configDetail:e}}catch(t){return null}}static async#o(e,r){try{if(t.#s.has(e))return null;let a=`${appConfig.cfg.urlPrefix}/template`;a=`${a}?tid=${e}&code=${r}`;const s=await fetch(a);if(200!==s.status)return 403===s.status?logwarn(`Failed to get template ${e}: No privilege`):logwarn(`Failed to get template ${e}, status: ${s.status}`),t.#s.add(e),null;const i=await s.json();if(!i||0!==i.code||!i.data)return logwarn(`Failed to get template ${e}: ${i.message}`),null;if(i.data.templateId!==e)return logerr(`Different templateId ${e} / ${i.templateId}`),null;const{domainId:n=0,defaultElementSource:o,configDetail:c="",maxExecutionDuration:l}=i.data;if(!c)return logerr(`Invalid content in template ${e}`),null;return{templateId:e,domainId:n,defaultElementSource:o,maxExecutionDuration:l,configDetail:c}}catch(t){return logdbg(t),null}}static async parseXmlTemplate(t,e=appConfig.cfg.useNickName){const r=await TaskParser.convertXmlToJson(t),a=TaskParser.getParaCfgsFromJsonCfg(r),s=TaskParser.getPartOfJsonCfg(r,"fonts",!0),i=TaskParser.getPartOfJsonCfg(r,"actions",!1),n=TaskParser.getAttrsInXml(r),o=TaskParser.getCaptchTypes(r),c=e?TaskParser.getDatableMapFromJsonCfg(r):null,l=(0,import_utils2.getCurrentUnixTime)();return{actionConfigs:i,paraCfgs:a,fontsConfig:s||null,attrsInXml:n,captchaTypes:o,datatableMap:c,lastUsedTime:l,lastCheckTime:l}}static async getTemplateConfig(e,r=""){if(!r){const r=t.#a.get(e);if(r)return r}let a=await t.#n(e,r);if(!a){const r=t.#i(e);a=await t.#o(e,r)}if(!a)throw new Error(`Cannot get template config of templateId ${e}`);const s=await t.parseXmlTemplate(a.configDetail),i=Object.assign({},s,{template:a});return t.#a.set(e,i),i}static clearTemplateConfig(e=0){return e>0?t.#a.delete(e):t.#a.clear(),!0}},import_utils11=require("@letsscrapedata/utils"),import_node_events=__toESM(require("events"),1),import_node_events2=require("events"),fs4=__toESM(require("fs"),1),path4=__toESM(require("path"),1),import_utils9=require("@letsscrapedata/utils"),import_controller2=require("@letsscrapedata/controller"),uuid=__toESM(require("uuid"),1),SysParas=class{#c;#l;#p;#g;constructor(t,e,r,a){this.#c=t,this.#l=e,this.#p=r,this.#g=a}get version(){return this.#g}get templateId(){return this.#l}get taskId(){return this.#p}get url(){try{return this.#c?.page?this.#c.page.url():""}catch{return""}}get hasPopupPage(){return this.#c?.popupPage?"1":"0"}get subtaskNum(){const t=this.#c?.subtasks;return Array.isArray(t)?String(t.length):"0"}get datatableNum(){const t=this.#c?.execData;return t&&"object"==typeof t?String(Object.keys(t).length):"0"}get responses(){try{const t=this.#c?.responses;return Array.isArray(t)?JSON.stringify(t):""}catch(t){return""}}get currentDate(){return(new Date).toISOString().slice(0,10)}get currentDateTime(){return(new Date).toISOString().slice(0,19)}get currentIsoTime(){return(new Date).toISOString()}get currentLocalTime(){const t=new Date;return`${t.getFullYear()}-${String(t.getMonth()+1).padStart(2,"0")}-${String(t.getDate()).padStart(2,"0")} ${String(t.getHours()).padStart(2,"0")}:${String(t.getMinutes()).padStart(2,"0")}:${String(t.getSeconds()).padStart(2,"0")}`}get currentUtcTime(){return(new Date).toISOString().slice(0,19).replace("T"," ")}get currentTime(){return(new Date).toISOString().slice(11,19)}get currentUnixMs(){return String(Date.now())}get currentUnixSec(){return String(Math.floor(Date.now()/1e3))}get random10(){return String(Math.floor(10*Math.random()))}get random100(){return String(Math.floor(100*Math.random()))}get random1000(){return String(Math.floor(1e3*Math.random()))}get random10000(){return String(Math.floor(1e4*Math.random()))}get uuidv1(){return uuid.v1()}get uuidv4(){return uuid.v4()}},iconv=__toESM(require("iconv-lite"),1),zlib=__toESM(require("zlib"),1),cheerio=__toESM(require("cheerio"),1),import_utils3=require("@letsscrapedata/utils"),LsdString=class t{static specialConvertFunObj={};static setSpecialConvertFunObj(e){return t.specialConvertFunObj=e,!0}static transformStr(e,r,a,s){let i=e;try{if("string"!=typeof e||!Array.isArray(r))throw new Error("Invalid paras in transformStr");for(const e of r){if("object"!=typeof e)throw new Error("Invalid funCfg in transformStr");const r="c"===e.type?`${e.type}_${e.subtype}`:e.type;let a=t._funPerformers[r];!a&&e.type.match(/^myfun[0-9]$/)&&"object"==typeof s&&(a=s[e.type]),a&&"function"==typeof a?i=a(i,e):logerr(`Invalid funName ${e.type} in transformStr`)}}catch(t){loginfo(t)}return i||a}static __getElementBySelector(t,e,r=0){try{const a="root"===e?t.root():t(e);return a.length<=0||-9999===r?a:a.eq(r)}catch(t){return logerr("##te  cheerio  invalid $ or loc"),null}}static __cheerioOptions={xmlMode:!0,decodeEntities:!1};static _c_attr(e,r){try{const{loc:a,idx:s,attrname:i}=r,n=parseInt(s),o=cheerio.load(e,t.__cheerioOptions);return t.__getElementBySelector(o,a,n).attr(i)}catch(t){return logerr(`##tmp cheerio c_attr err: ${t}`),""}}static _c_data(e,r){try{const{loc:a,idx:s,attrname:i}=r,n=parseInt(s),o=cheerio.load(e,t.__cheerioOptions);return t.__getElementBySelector(o,a,n).data(i)}catch(t){return logerr(`##tmp cheerio c_data err: ${t}`),""}}static _c_html(e,r){try{const{loc:a,idx:s}=r,i=parseInt(s),n=cheerio.load(e,t.__cheerioOptions);return t.__getElementBySelector(n,a,i).html()}catch(t){return logerr(`##tmp cheerio c_html err: ${t}`),""}}static _c_length(e,r){try{const{loc:a}=r,s=-9999,i=cheerio.load(e,t.__cheerioOptions),n=t.__getElementBySelector(i,a,s);return String(n.length)}catch(t){return logerr(`##tmp cheerio c_length err: ${t}`),"0"}}static _c_text(e,r){try{const{loc:a,idx:s}=r,i=parseInt(s),n=cheerio.load(e,t.__cheerioOptions);return t.__getElementBySelector(n,a,i).text()}catch(t){return logerr(`##tmp cheerio c_text err: ${t}`),""}}static _ceil(t){return String(Math.ceil(Number(t)))}static _closingsubstr(t,e){const{openchar:r,startstr:a,position:s}=e;if("string"!=typeof t||"string"!=typeof r||!["{","[","(","lt"].includes(r)||"string"!=typeof a||"number"!=typeof parseInt(s)||parseInt(s)<0)return"";const i="lt"===r?"<":r,n={"{":"}","[":"]","(":")","<":">"}[i];let o=1,c=-1;if(a){if(c=t.indexOf(a),c<0)return""}else c=parseInt(s);const l=t.indexOf(i,c);if(l<0)return-1;let p=l+1;for(;p<t.length;p++){let e=t[p];if(e===i)o++;else if(e===n&&(o--,0===o))break}const g=p<=t.length?p+1:-1;return g>l?t.substring(l,g):""}static __getCompressOptions(t,e){return t||e?void 0:{}}static _compress(e,r){try{const{method:a,sourceencoding:s,targetencoding:i,start:n,end:o}=r,c=t.__getCompressOptions(r,!0),l=Buffer.from(e,s);let p=null;return"gzip"===a?p=zlib.gzipSync(l,c):"deflate"===a?p=zlib.deflateSync(l,c):"deflateraw"===a?p=zlib.deflateRawSync(l,c):"brotli"===a&&(p=zlib.brotliCompressSync(l,c)),p?t.__convertBufferToString(p,i,n,o):""}catch(t){return""}}static _decompress(e,r){try{const{method:a,sourceencoding:s,targetencoding:i,start:n,end:o}=r,c=t.__getCompressOptions(r,!1),l=Buffer.from(e,s);let p=Buffer.from(e,s);return"gunzip"===a?p=zlib.gunzipSync(l,c):"inflate"===a?p=zlib.inflateSync(l,c):"inflateraw"===a?p=zlib.inflateRawSync(l,c):"unzip"===a?p=zlib.unzipSync(l,c):"brotli"===a&&(p=zlib.brotliDecompressSync(l,c)),p?t.__convertBufferToString(p,i,n,o):""}catch(t){return""}}static _concat(t,e){let r=t;const{str1:a,str2:s,str3:i,str4:n,str5:o}=e;return[a,s,i,n,o].forEach(t=>{"string"==typeof t&&(r=r.concat(t))}),r}static _specialconvert(e,r){let a=e;try{const{method:s}=r;return"function"==typeof t.specialConvertFunObj[s]&&(a=t.specialConvertFunObj[s](e,r)),a}catch(t){return a}}static _decode(t,e){const{str1:r,val1:a,str2:s,val2:i,str3:n,val3:o,str4:c,val4:l,str5:p,val5:g,defaultval:h}=e;try{return t===r?a:t===s?i:t===n?o:t===c?l:t===p?g:h}catch(t){return""}}static __convertBufferToString(t,e,r,a){const s=r?Number(r):0,i=a?Number(a):0;return t instanceof Buffer?i>0?t.toString(e,s,i):s>0?t.toString(e,s):t.toString(e):""}static _convertencoding(e,r){const{sourceencoding:a,targetencoding:s,start:i,end:n}=r;try{const r=Buffer.from(e,a);return t.__convertBufferToString(r,s,i,n)}catch(t){return""}}static _floor(t){return String(Math.floor(Number(t)))}static _hashcode(t,e){const{hashmethod:r}=e;return import_utils3.LsdUrl.getHashCode(t,r)}static _includes(t,e){const{searchstr:r}=e;if("string"!=typeof r)throw new Error("Invalid searchstr in _contains");return String(Number(t.includes(r)))}static _insert(t,e){const{str:r}=e;if("string"!=typeof r)throw new Error("Invalid str in _insert");return r+t}static _itemstoobj(t,e){const{split:r,kvsplit:a,keytrim:s,keynows:i,valuetrim:n,valuenows:o}=e;if(!r||!a)return"{}";try{let e={};const c=t.split(new RegExp(r)),l=new RegExp(a);for(const t of c){const r=t.split(l);if(2!==r.length)continue;let a=s?r[0].trim():r[0];i&&(a=a.split("").filter(t=>t.trim()).join(""));let c=n?r[1].trim():r[1];o&&(c=c.split("").filter(t=>t.trim()).join("")),a&&(e[a]=c)}return JSON.stringify(e)}catch(t){return"{}"}}static _jsonparse(t,e){const{key1:r,key2:a,key3:s,key4:i,key5:n,key6:o,keys:c,length:l,srctype:p}=e;let g=null;try{if("script"===p)return"";if(g=JSON.parse(t),"object"!=typeof g||!g)return""}catch(t){return""}return g=(0,import_utils3.getMemberOfObject)(g,r,a,s,i,n,o),null==g?"":"string"==typeof g?g:"number"==typeof g?String(g):(c&&(g=Array.from(Object.keys(g))),l&&Array.isArray(g)?String(g.length):JSON.stringify(g))}static _length(t){return String(t.length)}static _matchall(t,e){try{const{pattern:r,flags:a,itemvalue:s,filter:i,start:n,end:o,resulttype:c,join:l}=e,p=new RegExp(r,a),g=t.matchAll(p);if("boolean"===c)return g?"1":"0";if(!g)return"";let h=[];for(const t of g)if("match[0]"===s)h.push(t[0]);else{let e=s;for(let r=0;r<t.length;r++)e=e.replaceAll(`match[${r}]`,t[r]);t.index>=0&&(e=e.replaceAll("match.index",String(t.index))),h.push(e)}if(i){const t=new RegExp(i);h=h.filter(e=>e.match(t))}const d=parseInt(n),f=parseInt(o);return 0===d&&0===f||(h=0===f?h.slice(d):h.slice(d,f)),"all"===c?h.join(l):"length"===c?String(h.length):""}catch(t){return loginfo(t),""}}static _max(t,e){try{const{split:r}=e,a=t.split(new RegExp(r)).filter(t=>t.trim()).map(t=>Number(t));return a.length>0?String(a.sort((t,e)=>t-e).pop()):""}catch(t){return loginfo(t),""}}static _min(t,e){try{const{split:r}=e,a=t.split(new RegExp(r)).filter(t=>t.trim()).map(t=>Number(t));return a.length>0?String(a.sort((t,e)=>t-e)[0]):""}catch(t){return loginfo(t),""}}static _numeq(t,e){const{num:r}=e;return"string"!=typeof t||"string"!=typeof r?"0":t.trim().match(/^\d+$/)&&r.trim().match(/^\d+$/)&&Number(t)===Number(r)?"1":"0"}static _numge(t,e){const{num:r}=e;return"string"!=typeof t||"string"!=typeof r?"0":t.trim().match(/^\d+$/)&&r.trim().match(/^\d+$/)&&Number(t)>=Number(r)?"1":"0"}static _numgt(t,e){const{num:r}=e;return"string"!=typeof t||"string"!=typeof r?"0":t.trim().match(/^\d+$/)&&r.trim().match(/^\d+$/)&&Number(t)>Number(r)?"1":"0"}static _numle(t,e){const{num:r}=e;return"string"!=typeof t||"string"!=typeof r?"0":t.trim().match(/^\d+$/)&&r.trim().match(/^\d+$/)&&Number(t)<=Number(r)?"1":"0"}static _numlt(t,e){const{num:r}=e;return"string"!=typeof t||"string"!=typeof r?"0":t.trim().match(/^\d+$/)&&r.trim().match(/^\d+$/)&&Number(t)<Number(r)?"1":"0"}static _notincludes(t,e){const{searchstr:r}=e;if("string"!=typeof r)throw new Error("Invalid str in _notcontains");return String(Number(!t.includes(r)))}static _padEnd(t,e){const{targetlen:r,padstr:a}=e,s=parseInt(r),i="string"==typeof a&&a.length>0?a:" ";return t.padEnd(s,i)}static _padStart(t,e){const{targetlen:r,padstr:a}=e,s=parseInt(r),i="string"==typeof a&&a.length>0?a:" ";return t.padStart(s,i)}static _parseFloat(t){return String(parseFloat(t))}static _parseInt(t){return String(parseInt(t))}static _randomarryidxes(t,e){const{join:r}=e;try{const e=parseInt(t);return[...Array(e).keys()].sort(()=>Math.random()-.5).join(r)}catch(t){return""}}static _replace(t,e){const{substr:r,newstr:a,regexp:s,flags:i}=e;try{const e=s;if("string"!=typeof r)throw new Error("Invalid substr in _replace");if("string"!=typeof a)throw new Error("Invalid newstr in _replace");return e?t.replace(RegExp(r,i),a):t.replace(r,a)}catch(e){return t}}static _round(t){return String(Math.round(Number(t)))}static __sort(t,e,r){let a;return a="string"===e?"asc"===r?t.sort((t,e)=>t.localeCompare(e)):t.sort((t,e)=>e.localeCompare(t)):"asc"===r?t.sort((t,e)=>parseFloat(t)-parseFloat(e)):t.sort((t,e)=>parseFloat(e)-parseFloat(t)),a}static _setcomplement(e,r){const{array2:a,split1:s,split2:i,sorttype:n,valtype:o,limit:c,join:l}=r;let p,g;try{p=new RegExp(s),g=new RegExp(i)}catch(t){throw new Error(`Invalid regexp split1: ${s}, split2: ${i} in _setcomplement`)}try{const r=parseInt(c);if(!["nosort","asc","desc"].includes(n)||!["string","number"].includes(o)||r<0||"string"!=typeof a||"string"!=typeof l)throw new Error(`Invalid limit: ${c}, sorttype: ${n}, or array2 in _setcomplement`);const s=e.split(p),i=a.split(g);let h=(0,import_utils3.getComplementOfTwoArray)(s,i);return["asc","desc"].includes(n)&&(h=t.__sort(h,o,n)),r>0?h.slice(0,r).join(l):h.join(l)}catch(t){return loginfo(t),e}}static _setdifference(e,r){const{array2:a,split1:s,split2:i,sorttype:n,valtype:o,limit:c,join:l}=r;let p,g;try{p=new RegExp(s),g=new RegExp(i)}catch(t){throw new Error(`Invalid regexp split1: ${s}, split2: ${i} in _setcomplement`)}try{const r=parseInt(c);if(!["nosort","asc","desc"].includes(n)||!["string","number"].includes(o)||r<0||"string"!=typeof a||"string"!=typeof l)throw new Error(`Invalid limit: ${c}, sorttype: ${n}, or array2 in _setcomplement`);const s=e.split(p),i=a.split(g);let h=(0,import_utils3.getDifferenceOfTwoArray)(s,i);return["asc","desc"].includes(n)&&(h=t.__sort(h,o,n)),r>0?h.slice(0,r).join(l):h.join(l)}catch(t){return loginfo(t),e}}static _setintersection(e,r){const{array2:a,split1:s,split2:i,sorttype:n,valtype:o,limit:c,join:l}=r;let p,g;try{p=new RegExp(s),g=new RegExp(i)}catch(t){throw new Error(`Invalid regexp split1: ${s}, split2: ${i} in _setcomplement`)}try{const r=parseInt(c);if(!["nosort","asc","desc"].includes(n)||!["string","number"].includes(o)||r<0||"string"!=typeof a||"string"!=typeof l)throw new Error(`Invalid limit: ${c}, sorttype: ${n}, or array2 in _setcomplement`);const s=e.split(p),i=a.split(g);let h=(0,import_utils3.getIntersectionOfTwoArray)(s,i);return["asc","desc"].includes(n)&&(h=t.__sort(h,o,n)),r>0?h.slice(0,r).join(l):h.join(l)}catch(t){return loginfo(t),e}}static _setunion(e,r){const{array2:a,split1:s,split2:i,sorttype:n,valtype:o,limit:c,join:l}=r;let p,g;try{p=new RegExp(s),g=new RegExp(i)}catch(t){throw new Error(`Invalid regexp split1: ${s}, split2: ${i} in _setcomplement`)}try{const r=parseInt(c);if(!["nosort","asc","desc"].includes(n)||!["string","number"].includes(o)||r<0||"string"!=typeof a||"string"!=typeof l)throw new Error(`Invalid limit: ${c}, sorttype: ${n}, or array2 in _setcomplement`);const s=e.split(p),i=a.split(g);let h=(0,import_utils3.getUnionOfTwoArray)(s,i);return["asc","desc"].includes(n)&&(h=t.__sort(h,o,n)),r>0?h.slice(0,r).join(l):h.join(l)}catch(t){return loginfo(t),e}}static _shuffle(t,e){try{const{split:r}=e;if("string"!=typeof r)throw new Error("Invalid split in _shuffle");const a=t.split(new RegExp(r));return!Array.isArray(a)||a.length<=1?t:a.sort(()=>Math.random()-.5).join(r)}catch(e){return loginfo(e),t}}static _slice(t,e){const{beginidx:r,endidx:a}=e,s=parseInt(r),i="string"==typeof a&&a.length>0?parseInt(a):void 0;return t.slice(s,i)}static _sort(t,e){try{const{split:r,valtype:a,sorttype:s,join:i}=e;if("string"!=typeof r||!["string","number"].includes(a)||!["asc","desc"].includes(s))throw new Error("Invalid split or valtype in _sort");const n=t.split(new RegExp(r));return!Array.isArray(n)||n.length<=1?t:"string"===a?"asc"===s?n.sort((t,e)=>t.localeCompare(e)).join(i):n.sort((t,e)=>e.localeCompare(t)).join(i):"asc"===s?n.sort((t,e)=>parseFloat(t)-parseFloat(e)).join(i):n.sort((t,e)=>parseFloat(e)-parseFloat(t)).join(i)}catch(t){return loginfo(t),""}}static _split(t,e){try{const{split:r,limit:a,filter:s,start:i,end:n,resulttype:o,val:c,insertstr:l,join:p}=e;if("string"!=typeof r)throw new Error("Invalid split in _split");const g=new RegExp(r),h=parseInt(a);let d=h>0?t.split(g,h):t.split(g);if(!Array.isArray(d))return"";if(s){const t=new RegExp(s);d=d.filter(e=>e.match(t))}const f=parseInt(i),u=parseInt(n);if(0===f&&0===u||(d=0===u?d.slice(f):d.slice(f,u)),l&&(d=d.map(t=>`${l}${t}`)),"all"===o)return d.join(p);if("length"===o)return String(d.length);if("index"===o){const t=d.findIndex(t=>t.trim()===c);return t>=0?String(t+1):""}return""}catch(t){return loginfo(t),""}}static _streq(t,e){const{str:r}=e;return"string"!=typeof r?"0":t===r?"1":"0"}static _substrAfter(t,e){const{substr:r,position:a,last:s,include:i}=e,n=i,o=parseInt(a);if("string"!=typeof r)throw new Error("Invalid substr in _substrAfter");let c=0;return c=s?o?t.lastIndexOf(r,o):t.lastIndexOf(r):o?t.indexOf(r,o):t.indexOf(r),-1===c?t:String(n?t.substring(c):t.substring(c+r.length))}static _substrBefore(t,e){const{substr:r,position:a,last:s,include:i}=e,n=parseInt(a),o=i;if("string"!=typeof r)throw new Error("Invalid substr in substrBefore");let c=0;return c=s?n?t.lastIndexOf(r,n):t.lastIndexOf(r):n?t.indexOf(r,n):t.indexOf(r),-1===c?t:o?t.substring(0,c+r.length):t.substring(0,c)}static _toBool(t){try{let e=t.replace(/true/g,"1");return e=e.replace(/false/g,"0"),e.search(/^[\s\d.+\-*/()<>=!|&:?]*$/)>-1&&-1===e.search(/=>/)?String(Number(Boolean((0,eval)(e)))):(loginfo(`Invalid logic expression ${t} in _toBool`),!1)}catch(t){return!1}}static _toLowerCase(t){return t.toLowerCase()}static _toNum(t){let e=t;return e.search(/^[\s\d.+\-*/()<>=!|&:?]+$/)>-1&&-1===e.search(/=>/)&&e.trim().length>0?String(Number((0,eval)(e))):(loginfo(`Invalid arithmetic expression ${e} in _toNum`),"0")}static _toUpperCase(t){return t.toUpperCase()}static _trim(t){return t.trim()}static _trimEnd(t){return t.trimEnd()}static _trimStart(t){return t.trimStart()}static _uniq(t,e){try{const{split:r,join:a}=e;if("string"!=typeof r||"string"!=typeof a)throw new Error("Invalid split in _uniq");const s=t.split(new RegExp(r));return!Array.isArray(s)||s.length<=1?t:[...new Set(s)].join(a)}catch(e){return loginfo(e),t}}static _urlencode(t,e){try{const{method:r,encoding:a,percent:s}=e;if("URI"===r)return encodeURI(t);if("URIComponent"===r)return encodeURIComponent(t);if("iconv"===r){const e=iconv.encode(t,a).toString("hex").toUpperCase();if(!e||e.length%2!=0)return logerr("Invalid str length in _urlencode"),"";if(s){const t=e.match(/../g);return t?t.map(t=>`%${t}`).join(""):""}return e}return logerr(`Invalid method ${r} in _urlencode`),""}catch(t){return logerr(t),""}}static _urldecode(t,e){try{const{method:r,encoding:a,percent:s}=e;if("URI"===r)return decodeURI(t);if("URIComponent"===r)return decodeURIComponent(t);if("iconv"===r){const e=s?/.../g:/../g,r=t.match(e);if(!r)return"";const i=r.map(t=>parseInt(t.slice(1),16));return iconv.decode(Buffer.from(i),a)}return logerr(`Invalid method ${r} in _urlencode`),""}catch(t){return logerr(t),""}}static _urldelparams(t,e){const{param1:r,param2:a,param3:s}=e;if("string"!=typeof r)throw new Error("Invalid params in _urldel");let i="";try{const e=new URL(t);if(!e||!e.searchParams||!e.searchParams.delete)return"";r&&e.searchParams.delete(r),a&&e.searchParams.delete(a),s&&e.searchParams.delete(s),i=e.href}catch(t){i=""}return i||""}static _urlget(t,e){const{attrorpath:r,hashmethod:a,param:s,hostnameparts:i,params:n,title:o,extname:c,validextnames:l}=e;if("string"!=typeof r)throw new Error("Invalid attr in _urlget");let p="";try{p=["href","origin","protocol","username","password","host","search","hash","hostname","port","hrefwithoutsearch","param"].includes(r)?import_utils3.LsdUrl.getUrlAttribute(t,r,i,s):import_utils3.LsdUrl.getFilePathFromUrl(t,r,a,i,n,c,l,o)}catch(t){p=""}return"string"==typeof p?p:""}static __setUrlParam(t,e,r,a){if(!t||!t.searchParams||!t.searchParams.get||"string"!=typeof e||"string"!=typeof r)return!1;if(!e)return!0;if(!a){if(null!==t.searchParams.get(e))return!0}return t.searchParams.set(e,r),!0}static _urlsetparams(e,r){const{param1:a,val1:s,param2:i,val2:n,param3:o,val3:c,replace:l}=r,p=l;if("string"!=typeof a)throw new Error("Invalid params in _urlset");let g="";try{const r=new URL(e);t.__setUrlParam(r,a,s,p),t.__setUrlParam(r,i,n,p),t.__setUrlParam(r,o,c,p),g=r.href}catch(t){g=""}return g||""}static _funPerformers={c_attr:t._c_attr,c_data:t._c_data,c_html:t._c_html,c_length:t._c_length,c_text:t._c_text,ceil:t._ceil,closingsubstr:t._closingsubstr,compress:t._compress,concat:t._concat,convertencoding:t._convertencoding,decode:t._decode,decompress:t._decompress,floor:t._floor,hashcode:t._hashcode,includes:t._includes,insert:t._insert,itemstoobj:t._itemstoobj,jsonparse:t._jsonparse,length:t._length,matchall:t._matchall,max:t._max,min:t._min,numeq:t._numeq,numge:t._numge,numgt:t._numgt,numle:t._numle,numlt:t._numlt,notincludes:t._notincludes,padend:t._padEnd,padstart:t._padStart,parsefloat:t._parseFloat,parseint:t._parseInt,randomarryidxes:t._randomarryidxes,replace:t._replace,round:t._round,setcomplement:t._setcomplement,setdifference:t._setdifference,setintersection:t._setintersection,setunion:t._setunion,shuffle:t._shuffle,slice:t._slice,sort:t._sort,specialconvert:t._specialconvert,split:t._split,streq:t._streq,substrafter:t._substrAfter,substrbefore:t._substrBefore,tobool:t._toBool,tolowercase:t._toLowerCase,tonum:t._toNum,touppercase:t._toUpperCase,trim:t._trim,trimend:t._trimEnd,trimstart:t._trimStart,uniq:t._uniq,urldecode:t._urldecode,urldelparams:t._urldelparams,urlencode:t._urlencode,urlget:t._urlget,urlsetparams:t._urlsetparams}};function getTransformExpStr(t,e,r="",a={}){return LsdString.transformStr(t,e,r,a)}var fs2=__toESM(require("fs"),1),path2=__toESM(require("path"),1),import_got_scraping=require("got-scraping"),import_header_generator=require("header-generator"),import_dateformat=__toESM(require("dateformat"),1),uuid2=__toESM(require("uuid"),1),import_utils4=require("@letsscrapedata/utils"),import_hpagent=require("hpagent"),download=__toESM(require("download"),1);async function lsdDownload(t,e="",r=null){const a={};if(e){const t=new import_hpagent.HttpsProxyAgent({keepAlive:!0,keepAliveMsecs:1e3,maxSockets:256,maxFreeSockets:256,scheduling:"lifo",proxy:e});a.agent={https:t}}let s;return r&&(a.headers=r),s=e||r?await download.default(t,void 0,a):await download.default(t),s}var GetPara=class t{static getSpecialFunObj={};static setGetSpecialFunObj(e){return t.getSpecialFunObj=e,!0}static getOtherFunObj={};static setGetOtherFunObj(e){return t.getOtherFunObj=e,!0}static noNeedToReturnContent="";static async getPara(e,r){if(!e||!e.type||!r)return"";const a=e.type;let s="";switch(a){case"addedtaskparas":"function"==typeof t.getOtherFunObj.getAddedTaskParasFun&&(s=await t.getOtherFunObj.getAddedTaskParasFun(e));break;case"api":s=await t.getApi(e,r);break;case"datetime":s=t.getDatetime(e);break;case"execedtaskparas":"function"==typeof t.getOtherFunObj.getExecedTaskParasFun&&(s=await t.getOtherFunObj.getExecedTaskParasFun(e));break;case"file":s=await t.getFile(e,r);break;case"httpheaders":s=await t.getHttpHeaders(e,r);break;case"queuedcntwithparas":"function"==typeof t.getOtherFunObj.getQueuedCntWithParasCfgFun&&(s=await t.getOtherFunObj.getQueuedCntWithParasCfgFun(e));break;case"queuedtaskparas":"function"==typeof t.getOtherFunObj.getQueuedTaskParasFun&&(s=await t.getOtherFunObj.getQueuedTaskParasFun(e));break;case"random":s=t.getRandom(e);break;case"response":s=await t.getResponse(e,r);break;case"special":s=await t.getSpecial(e,r);break;case"sysparas":s="please use ${sysParas.paraname}!!!";break;default:(0,import_utils4.unreachable)(a)}return s}static async getParaInElement(e,r,a){if(!e||!e.type||!a)return"";const s=e.type,i=a.paras;let n="";if("screenshot"===s)n=await t.getScreenshot(r,e,i);else(0,import_utils4.unreachable)(s);return n}static async getParaInPage(e,r,a){if(!e||!e.type||!a)return"";const s=e.type,i=a.paras;let n="";switch(s){case"content":n=await t.getContent(r);break;case"cookies":n="";break;case"mhtml":n=await t.getMhtml(r,e,i);break;case"pdf":n=await t.getPdf(r,e,i);break;case"screenshot":n=await t.getScreenshot(r,e,i);break;case"title":n=await t.getTitle(r,e);break;case"window":n=await t.getWindow(r,e);break;default:(0,import_utils4.unreachable)(s)}return n}static async getApi(e,r){const{method:a,url:s,encodeuri:i,session:n,headerssource:o,headers:c,referer:l,proxy:p,data:g,datatype:h}=e,d=e.context,f=e.timeout?Number(e.timeout):0;let u="";if(!["GET","DELETE","POST","PUT"].includes(a.toUpperCase()))return logerr(`##task  invalid method ${a} in getApi`),"";const m=s.startsWith("/");try{const e=i?encodeURI(s):s;if(m){if("function"==typeof t.getOtherFunObj.getDataFromServerFun){const r=f>0?{timeout:f}:void 0;return await t.getOtherFunObj.getDataFromServerFun(a,e,g,r)}return""}{const t={method:a.toUpperCase()};if(p){const e=r.proxyUrl?r.proxyUrl:"";e&&(t.proxyUrl=e)}n&&r.proxyUrl&&"default"===d&&(t.sessionToken=r.proxyUrl);const s="Content-Length";let i=!1,m={};if("headers"!==o||"headers"===o&&c||l){let a="";const n=r.interceptionData;if("headers"===o&&c?(m=JSON.parse(c),i="0"===m[s]):"cachedRequestHeaders"===o?(a=n.cachedRequestUrl,m=Object.assign({},n.cachedRequestHeaders),i=void 0!==m[s]):"cachedResponseHeaders"===o?(a=n.cachedResponseUrl,m=Object.assign({},n.cachedResponseHeaders),i=void 0!==m[s]):"origStateHeaders"===o?(a=n.origStateUrl,m=Object.assign({},n.origStateHeaders),i=void 0!==m[s]):"updatedRequestHeaders"===o?(a=n.updatedRequestUrl,m=Object.assign({},n.updatedRequestHeaders),i=void 0!==m[s]):"updatedResponseHeaders"===o?(a=n.updatedResponseUrl,m=Object.assign({},n.updatedResponseHeaders),i=void 0!==m[s]):"updatedStateHeaders"===o&&(a=n.updatedStateUrl,m=Object.assign({},n.updatedStateHeaders),i=void 0!==m[s]),o.endsWith("Headers")){const t=new URL(e),r=new URL(a);if(t.origin!==r.origin){if(!["origStateHeaders","updatedStateHeaders"].includes(o))return logerr(`Differenct url origins in getApi: req ${t.origin}, orig ${r.origin} `),"";if(!t.hostname.endsWith(r.hostname))return logerr(`Invalid url hostname in getApi: req ${t.hostname}, orig ${r.hostname} `),""}}l&&(m.referer=l),t.headers=m}if(g)if(i&&(m[s]=String(g.length)),"browser"===d||"state"===d||"task"===d||"standalone"===d)"json"===h?t.data=JSON.parse(g):"form"===h?t.form=JSON.parse(g):t.data=g;else if("default"===d)"json"===h?t.data=JSON.parse(g):"form"===h?t.form=JSON.parse(g):t.body=g;else if("fetch"===d)if("json"===h)t.data=JSON.parse(g);else if("form"===h){const e=new FormData,r=JSON.parse(g);for(const t of Object.keys(r))e.set(t,r[t]);t.data=e}else t.body=g;else logerr(`Invalid context ${d} in getApi`);if("browser"===d||"state"===d||"task"===d||"standalone"===d){const a=r.apiContext;if(!a)throw new Error("Null ApiContext in getApi");f>0&&(t.timeout=f);u=(await a.fetch(e,t)).text}else if("default"===d){f>0&&(t.timeout={request:f});u=(await(0,import_got_scraping.gotScraping)(e,t)).body}else if("fetch"===d){const r=await fetch(e,t);u=await r.text()}else logerr(`Invalid context ${d} in getApi`);return"string"==typeof u?u:JSON.stringify(u)}}catch(t){return loginfo(t),""}}static async getContent(t){try{if(!t)return"";return await t.content()}catch(t){return""}}static async getCookies(t,e){try{if(!t)return"";const{urls:r,domain:a,name:s,path:i,value:n}=e;let o="";r.length>0&&r.split(";");let c=await t.cookies();return a&&c.filter(t=>t.domain===a),s&&c.filter(t=>t.name===s),i&&c.filter(t=>t.path===i),o=n?c.length>0?c[0].value:"":JSON.stringify(c),o}catch(t){return""}}static getDatetime(t){const{format:e}=t;return(0,import_dateformat.default)(new Date,e)}static async getFile(e,r){const a=r.paras;try{const{url:s,pathtype:i,hashmethod:n,hostnameparts:o,params:c,extname:l,validextnames:p,basedir:g,setvar:h,pathvarname:d,proxy:f,headers:u,referer:m}=e;let y=e.path.trim();if(y)y=path2.join(y);else{let t=import_utils4.LsdUrl.getFilePathFromUrl(s,i,n,Number(o),c,l,p);y=path2.join(g,t)}if(y){const t=path2.dirname(y);if(!await(0,import_utils4.fsCheckOrCreateSubdir)(appConfig.cfg.baseFileDir,t))throw new Error("failed to checkOrCreateFileDir");y=path2.join(appConfig.cfg.baseFileDir,y)}const C=f&&r.proxyUrl?r.proxyUrl:"";let w=null;(u||m)&&(w=u?JSON.parse(u):{},m&&(w.referer=m));const k=await lsdDownload(s,C,w);return fs2.writeFileSync(y,k),d&&a&&!appConfig.RESERVED_PARA_NAMES.includes(d)&&(a[d]=y),h?k.toString("utf8"):t.noNeedToReturnContent}catch(t){throw new Error("error in get_file")}}static _genHeaderMethod={genLatestAndroid:import_header_generator.PRESETS.MODERN_ANDROID,genLatestDesktop:import_header_generator.PRESETS.MODERN_DESKTOP,genLatestLinux:import_header_generator.PRESETS.MODERN_LINUX,genLatestLinuxChrome:import_header_generator.PRESETS.MODERN_LINUX_CHROME,genLatestLinuxFirefox:import_header_generator.PRESETS.MODERN_LINUX_FIREFOX,genLatestMacos:import_header_generator.PRESETS.MODERN_MACOS,genLatestMacosChrome:import_header_generator.PRESETS.MODERN_MACOS_CHROME,genLatestMacosFirefox:import_header_generator.PRESETS.MODERN_MACOS_FIREFOX,genLatestMobile:import_header_generator.PRESETS.MODERN_MOBILE,genLatestWindows:import_header_generator.PRESETS.MODERN_WINDOWS,genLatestWindowsChrome:import_header_generator.PRESETS.MODERN_WINDOWS_CHROME,genLatestWindowsFirefox:import_header_generator.PRESETS.MODERN_WINDOWS_FIREFOX};static async getHttpHeaders(e,r){const{headermethod:a,origheaders:s}=e,{genHeaderOptions:i,newHttpHeaders:n}=r,o=r.interceptionData;let c={},l=!1;try{if(t._genHeaderMethod[a]){l=!0;c=new import_header_generator.HeaderGenerator(t._genHeaderMethod[a]).getHeaders()}else if("genNewByHttpOptions"===a){l=!0;c=new import_header_generator.HeaderGenerator(i).getHeaders()}else if("origheaders"===a)s&&(c=JSON.parse(s));else{if(!o)throw new Error("logic error: no interceptionData");"cachedRequestHeaders"===a?(c=Object.assign({},o.cachedRequestHeaders),o.updatedRequestUrl=o.cachedRequestUrl,o.updatedRequestHeaders=c):"cachedResponseHeaders"===a?(c=Object.assign({},o.cachedResponseHeaders),o.updatedResponseUrl=o.cachedResponseUrl,o.updatedResponseHeaders=c):"origStateHeaders"===a?(c=Object.assign({},o.origStateHeaders),o.updatedStateUrl=o.origStateUrl,o.updatedStateHeaders=c):"updatedRequestHeaders"===a?c=o.updatedRequestHeaders:"updatedResponseHeaders"===a?c=o.updatedResponseHeaders:"updatedStateHeaders"===a?c=o.updatedStateHeaders:logerr(`##get invalid httpMethod ${a} in getHttpHeaders`)}if(n)for(const t of Object.values(n)){const{name:e,source:r,value:s,deleteFlag:i}=t;if(i)delete c[e];else if("value"===r)c[e]=s;else if(!l&&r!==a){let t={};if("cachedRequestHeaders"===r)t=o.cachedRequestHeaders;else if("cachedResponseHeaders"===r)t=o.cachedResponseHeaders;else{if("origStateHeaders"!==r){logerr(`Invalid soure ${r} in httpheader`);continue}t=o.origStateHeaders}"string"==typeof t[e]?c[e]=t[e]:s||(c[e]=s)}}return l?JSON.stringify(c):""}catch(t){return logerr(t),""}}static async getMhtml(t,e,r){try{if(!t)return"";let a=e.path.trim();const{pathtype:s,hashmethod:i,hostnameparts:n,basedir:o,pathvarname:c}=e;if(a)a=path2.join(a);else{const e=t.url();let r=await t.title();a=import_utils4.LsdUrl.getFilePathFromUrl(e,s,i,Number(n),"","mhtml","",r),a=path2.join(o,a)}const{data:l}=await t.sendCDPMessage("Page.captureSnapshot",{format:"mhtml"});if(a){const t=path2.dirname(a);if(!await(0,import_utils4.fsCheckOrCreateSubdir)(appConfig.cfg.baseFileDir,t))throw new Error("failed to checkOrCreateMhtmlDir");const e=path2.join(appConfig.cfg.baseFileDir,a);fs2.writeFileSync(e,l),c&&r&&!appConfig.RESERVED_PARA_NAMES.includes(c)&&(r[c]=a)}return String(l)}catch(t){throw logdbg(t),new Error("error in get_mhtml")}}static _getScreenSize(t){return String(Number(t))===t?`${t}px`:t}static async getPdf(e,r,a){try{if(!e)return"";const{pathtype:s,hashmethod:i,hostnameparts:n,basedir:o,scale:c,displayheaderfooter:l,headertemplate:p,footertemplate:g,printbackground:h,landscape:d,pageranges:f,format:u,width:m,height:y,top:C,right:w,bottom:k,left:_,onepage:v,hmargin:S,pathvarname:b,setvar:x}=r;let E={},T=r.path.trim();if(T)T=path2.join(T);else{const t=e.url();let r=await e.title();T=import_utils4.LsdUrl.getFilePathFromUrl(t,s,i,Number(n),"","pdf","",r),T=path2.join(o,T)}if(T){const t=path2.dirname(T);if(!await(0,import_utils4.fsCheckOrCreateSubdir)(appConfig.cfg.baseFileDir,t))throw new Error("failed to checkOrCreatePdfDir");T=path2.join(appConfig.cfg.baseFileDir,T),E.path=T}if("1"!==c&&(E.scale=Number(c)),l&&(E.displayHeaderFooter=l),p&&(E.headerTemplate=p),g&&(E.footerTemplate=g),h&&(E.printBackground=h),d&&(E.landscape=d),f&&(E.pageRanges=f),C||w||k||_||l){const e={};C?e.top=t._getScreenSize(C):l&&!p&&(e.top="60px"),k?e.bottom=t._getScreenSize(k):l&&!g&&(e.bottom="60px"),w&&(e.right=t._getScreenSize(w)),_&&(e.left=t._getScreenSize(_)),E.margin=e}if(v){const t=await e.pageWidth();let r=await e.pageHeight();S&&(r+=Number(S)),E.width=`${t}px`,E.height=`${r}px`,logdbg(`--------- pageHeight：${r}px`)}else m||y?(m&&(E.width=t._getScreenSize(m)),y&&(E.height=t._getScreenSize(y))):"Letter"!==u&&(E.format=u);const A=await e.pdf(E);return b&&a&&!appConfig.RESERVED_PARA_NAMES.includes(b)&&(a[b]=T),x?A.toString("base64"):t.noNeedToReturnContent}catch(t){throw logdbg(t),new Error("error in get_pdf")}}static getRandom(t){try{const{min:e,max:r}=t,a=parseInt(e),s=parseInt(r);if(a>=s)return"0";const i=Math.round(Math.random()*(s-a))+a;return String(i)}catch(t){return"0"}}static async getResponse(t,e){try{const{requrl:r,idx:a,length:s}=t;let i=e?.interceptionData?.responses;if(!Array.isArray(i)||i.length<=0||!i.every(t=>t.requestUrl))return"";if(r){const t=new RegExp(r);i=i.filter(e=>e.requestUrl.match(t))}const n=i.length;if(n<=0)return"";if(s)return String(n);const o=parseInt(a);let c="";return o>=0&&o<n?c=i[o]?.responseData:o<0&&o+n>=0&&(c=i.slice(o)[0]?.responseData),"string"==typeof c?c:""}catch(t){throw new Error("error in get_response")}}static async getScreenshot(e,r,a){let s={};try{if(!e)return"";let{basedir:i,path:n,extname:o,quality:c,fullpage:l,x:p,y:g,width:h,height:d,omitbackground:f,pathvarname:u,setvar:m}=r;if(n=n.trim(),n)n=path2.join(i,n);else if(e.title){const t=await e.title(),r=`${import_utils4.LsdUrl.convertToValidPathname(t)}.${o}`;n=path2.join(i,r)}else{const t=`${uuid2.v4()}.${o}`;n=path2.join(i,t)}if(n){const t=path2.dirname(n);if(!await(0,import_utils4.fsCheckOrCreateSubdir)(appConfig.cfg.baseFileDir,t))throw new Error("failed to checkOrCreatePdfDir");n=path2.join(appConfig.cfg.baseFileDir,n),s.path=n}"jpeg"===o&&(s.type="jpeg");const y=parseInt(c);"jpeg"===o&&y>=0&&(s.quality=y),l&&(s.fullPage=l);const C=parseInt(h),w=parseInt(d);if(C>0&&w>0){const t={x:parseInt(p),y:parseInt(g),width:C,height:w};s.clip=t}f&&(s.omitBackground=f);const k=await e.screenshot(s);return u&&a&&!appConfig.RESERVED_PARA_NAMES.includes(u)&&(a[u]=n),m&&k?k.toString("base64"):t.noNeedToReturnContent}catch(t){throw new Error("error in get_screenshot")}}static async getSpecial(e,r){let a="";const s=e.method;return"function"==typeof t.getSpecialFunObj[s]&&(a=await t.getSpecialFunObj[s](e,r)),a}static async getTitle(t,e){try{if(!t)return"";const{aspath:r}=e,a=await t.title();return r?import_utils4.LsdUrl.convertToValidPathname(a):a}catch(t){return""}}static async getWindow(t,e){try{const{key1:r,key2:a,key3:s,key4:i,key5:n,key6:o,keys:c,length:l}=e;if(!t||!r)return"";const p=[r,a,s,i,n,o];let g=await t.windowMember(p);if(!c&&!l)return g;const h=JSON.parse(g);return h&&"object"==typeof h?c?Array.from(Object.keys(h)).join(","):l&&Array.isArray(h)?String(h.length):"":""}catch(t){return logwarn(t),""}}},path3=__toESM(require("path"),1),fsp=__toESM(require("fs/promises"),1),import_utils5=require("@letsscrapedata/utils");async function responseCacheCb(t,e,r){if(e&&t)try{const{execData:a,contentType:s,tabname:i,responseFlag:n,requestHeadersFlag:o,responseHeadersFlag:c,interceptionData:l}=e,{pageUrl:p,cookies:g}=r,h=t.request(),d=h.method(),f=h.url();let u=h.postData();const m=u||"",y="text"===s?await t.text():"";if(i&&(a[i]||(a[i]=[]),a[i].push({c01:p,c02:d,c03:f,c04:m,c05:y,c06:String(y.length)})),n&&l.responses.push({pageUrl:p,requestMethod:d,requestUrl:f,requestData:m,responseData:y}),o){const t=Object.assign({},h.allHeaders?await h.allHeaders():h.headers());!t.cookie&&g&&(t.cookie=g),l.cachedRequestUrl=f,l.cachedRequestHeaders=t,logdbg("requestUrl: ",f),logdbg("requestHeaders: ",t)}if(c){const e=Object.assign({},t.allHeaders?await t.allHeaders():t.headers());l.cachedResponseUrl=f,l.cachedResponseHeaders=e,logdbg("responseHeaders: ",e)}}catch(t){logerr(t)}}var isHttpResponse=t=>void 0!==t.buffer;async function responseSaveCb(t,e,r){if(!e||!t)return;const a=t.request(),s=a.url(),{execData:i,tabname:n,pathType:o,hashMethod:c,baseDir:l,hostnameParts:p,params:g,extname:h,validExtnames:d,encoding:f}=e,{pageUrl:u}=r;let m=import_utils5.LsdUrl.getFilePathFromUrl(s,o,c,p,g,h,d);s.startsWith("http")||logdbg("@@response: ",t),logdbg("@@reqUrl: ",s.slice(0,100)),logdbg("@@filename: ",m);let y=path3.join(l,m);if(!y)return;const C=path3.dirname(y);if(!await(0,import_utils5.fsCheckOrCreateSubdir)(appConfig.cfg.baseFileDir,C))throw new Error("failed to checkOrCreateReponseDir");const w=path3.join(appConfig.cfg.baseFileDir,y),k=isHttpResponse(t)?await t.buffer():await t.body();let _=f;if(!f){const t=a.resourceType();_=["document","stylesheet","script","xhr","fetch"].includes(t)?"utf8":"binary"}if(await fsp.writeFile(w,k,_),n&&i){const t=a.method();let e=a.postData();const r={c01:u,c02:t,c03:s,c04:e||"",c05:y,c06:String(k.length)};i[n]?i[n].push(r):i[n]=[r]}}var import_utils7=require("@letsscrapedata/utils"),import_events=__toESM(require("events"),1),import_utils6=require("@letsscrapedata/utils"),TwoCaptcha=class{#h;#d;#f;#u;#m;#y;async#C(t,e,r="POST"){try{const a={method:r,headers:{"Content-Type":"application/json"},body:JSON.stringify(e)},s=await fetch(t,a);if(!s.ok)return logerr(`##captcha failed to fetch, url ${t}`),null;const i=await s.json();return console.log("----------------------- fetch res: ",i),i}catch(t){return logerr(t),null}}async#w(t){const e={clientKey:this.#h,task:t,softId:0};"string"==typeof t.lang&&t.lang&&(e.languagePool=t.lang,delete t.lang),console.log("-----------------------  createTask body: ",e);return await this.#C("https://api.2captcha.com/createTask",e)}async#k(t){const e={clientKey:this.#h,taskId:t};return await this.#C("https://api.2captcha.com/getTaskResult",e)}async#_(){const t={clientKey:this.#h},e=await this.#C("https://api.2captcha.com/getBalance",t);return e?.balance>=0?e.balance:-1}#v(t){return`event${t}`}#S(t){return 5e3}async#b(t,e){const r={errorCode:"SUCCEEDED",taskId:0,cost:0,ip:"",createTime:(0,import_utils6.getCurrentUnixTime)(),endTime:0,solveCount:0},{errorId:a,taskId:s}=await this.#w(t);if(a>0||s<=0)return r.errorCode="ERROR_OTHER",s>0&&(r.taskId=s),r;r.taskId=s;const i=this.#v(s);await(0,import_utils6.sleep)(this.#S(e));let n=0;this.#u||(n=setInterval(async()=>{const t=await this.#k(s);console.log(`--------------------------- getResult of taskId ${s} @${new Date}: `,t),0!==t?.errorId?this.#y.emit(i,{code:1001,message:"failed",data:t}):0===t.errorId&&"ready"===t.status&&this.#y.emit(i,{code:0,message:"succeeded",data:t})},this.#d)),setTimeout(()=>this.#y.emit(i,{code:1009,message:"timeout",data:null}),this.#f);const[o]=await(0,import_events.once)(this.#y,i);console.log(`--------------------------- get eventData of taskId ${s} @${new Date}: `,o),n&&clearInterval(n),r.endTime=(0,import_utils6.getCurrentUnixTime)();const{code:c,data:l}=o;if(0===c){if(!l||0!==l.errorId)throw new Error("##captcha invalid data in getResult");const{cost:t,ip:e,createTime:a,endTime:s,solveCount:i,solution:n}=l;t&&(r.cost=Number(t)),e&&(r.ip=e),a&&a>0&&(a-r.createTime>5&&loginfo("##captcha task createTime difference: "+(a-r.createTime)),r.createTime=a),s&&s>0&&(r.endTime-s>2*this.#d&&loginfo("##captcha task endTime difference: "+(r.endTime-s)),r.endTime=s),i&&i>0&&(r.solveCount=i),n&&(r.solution=n)}else if(1001===c)r.errorCode=l?.errorCode?l.errorCode:"ERROR_OTHER";else{if(1009!==c)throw new Error(`##captcha unreachable at ${(new Error).stack}`);r.errorCode="ERROR_TIMEOUT"}return r}#x(t,e,r){let a=r?e:`${e}Proxyless`;return r?Object.assign({},t,{type:a},r):Object.assign({},t,{type:a})}#E(t,e){const{version:r,websiteURL:a,gt:s,challenge:i,geetestApiServerSubdomain:n,userAgent:o,captchaId:c}=t,l="GeeTestTask",p={type:e?l:`${l}Proxyless`,version:r,websiteURL:a};return s&&(p.gt=s),i&&(p.challenge=i),n&&(p.geetestApiServerSubdomainapiDomain=n),c&&(p.initParameters={captcha_id:c}),o&&(p.userAgent=o),p}#T(t,e){const{version:r,websiteKey:a,websiteURL:s,isEnterprise:i,isInvisible:n,s:o,action:c,minScore:l,userAgent:p,cookies:g,apiDomain:h}=t;let d="RecaptchaV3TaskProxyless";if(2===r){const t=i?"RecaptchaV2EnterpriseTask":"RecaptchaV2Task";d=e?t:`${t}Proxyless`}const f={type:d,websiteURL:s,websiteKey:a};return h&&(f.apiDomain=h),f.isInvisible=n,o&&(i?f.enterprisePayload=o:f.recaptchaDataSValue=o),p&&(f.userAgent=p),g&&(f.cookies=g),f.isEnterprise=i,l&&(f.minScore=l),c&&(f.pageAction=c),f}#A(t,e){const r="TurnstileTask";let a=e?r:`${r}Proxyless`;const{websiteKey:s,websiteURL:i,action:n,cData:o,chlPageData:c}=t;return n?{type:a,websiteKey:s,websiteURL:i,action:n,data:o,pagedata:c}:{type:a,websiteKey:s,websiteURL:i}}#I(t){const{images:e,comment:r,lang:a,math:s=!1,maxLength:i=0,minLength:n=0,question:o=!1,space:c=!1,textType:l="any"}=t;if(o)return{type:"TextCaptchaTask",comment:r,lang:a};{let o=0;"numbers"===l?o=1:"letters"===l&&(o=2);const p={type:"ImageToTextTask",body:e[0].toString("base64"),case:!!t.case,comment:r,math:s,maxLength:i,minLength:n,numeric:o,phrase:c,lang:a};return e.length>1&&(p.imgInstructions=e[1].toString("base64")),p}}#O(t){const{images:e,comment:r,lang:a,maxClicks:s=0,minClicks:i=1}=t,n={type:"CoordinatesTask",body:e[0].toString("base64"),comment:r,lang:a,minClicks:i};return s>0&&s>=i&&(n.maxClicks=s),e.length>1&&(n.imgInstructions=e[1].toString("base64")),n}constructor(t){const{clientKey:e,callbackUrl:r="",pollingInterval:a=5e3,timeout:s=15e4}=t;if(!e)throw new Error("Invalid options: no clientKey");this.#h=e,this.#d=a>=2e3?a:2e3,this.#f=s>=3e4?s:3e4,this.#u=r||"",r&&console.log(this.#u),this.#m=["amazon","funcaptcha","geetest","keycaptcha","mtcaptcha","recaptcha","turnstile","text","coordinate","grid","slider","rotation"],this.#y=new import_events.default}canSolveCaptcha(t){return this.#m.includes(t)}async getBalance(){return await this.#_()}name(){return"2captcha"}sendResult(t){if(!t||"object"!=typeof t)return!1;const{taskId:e,errorId:r}=t;if(e>0){const e=this.#v(t.taskId);return 0===r?this.#y.emit(e,{code:0,message:"succeeded",data:t}):0===r&&this.#y.emit(e,{code:1001,message:"failed",data:t}),!0}return!1}async amazon(t,e){return await this.#b(this.#x(t,"AmazonTask",e),"amazon")}async funcaptcha(t,e){return await this.#b(this.#x(t,"FunCaptchaTask",e),"funcaptcha")}async geetest(t,e){return await this.#b(this.#E(t,e),"geetest")}async keycaptcha(t,e){return await this.#b(this.#x(t,"KeyCaptchaTask",e),"keycaptcha")}async mtcaptcha(t,e){return await this.#b(this.#x(t,"MtCaptchaTask",e),"mtcaptcha")}async recaptcha(t,e){return await this.#b(this.#T(t,e),"recaptcha")}async turnstile(t,e){return await this.#b(this.#A(t,e),"turnstile")}async text(t){return await this.#b(this.#I(t),"text")}async coordinate(t){return await this.#b(this.#O(t),"coordinate")}async grid(t){return await this.#b(this.#x(t,"TextTask"),"text")}async slider(t){return await this.#b(this.#x(t,"TextTask"),"text")}async rotation(t){return await this.#b(this.#x(t,"TextTask"),"text")}},solvers=[];function getSolver(t){if(0===solvers.length){if(!appConfig.cfg?.captcha?.clientKey)throw new Error("Cannot solve captcha, please input clientKey of captcha provider.");{const t=new TwoCaptcha({clientKey:appConfig.cfg.captcha.clientKey});solvers.push(t)}}for(const e of solvers)if(e.canSolveCaptcha(t))return e;throw new Error(`##captcha  cannot solve ${t}`)}async function executeTask(t){const e=getSolver(t.type);let r;if(!e[t.type])throw new Error("##captcha logic error");return r=await e[t.type](t),console.log("---------------- response: ",r),r}var processors=new Map,captchaTypeAttr="data-captcha-type",widgetIdAttr="data-widget-id",captchaOptions={rootTag:"head",parentTag:"capwgts",captchaTag:"capwgt",solutionAttr:"value"};function checkCaptchaType(t,e){if(t.captchaType!==e)throw new Error(`##captcha invalid captchaType ${t.captchaType}`);return!0}function isValidStringAttr(t){return"string"==typeof t&&!!t&&!["null","undefined"].includes(t)}async function getCaptchasOnce(t,e,r=captchaOptions){let a=[];if(e){const r={opType:"get"},s=await t.evaluate(async([t,e])=>{const r=window[t];return"function"==typeof r?r(e):""},[e,r]);return Array.isArray(s)?(a=s,a):(logerr(`##captcha  no function window.${e}`),[])}{const e=`${r.rootTag} > ${r.parentTag} > ${r.captchaTag}`,s=await t.findElements(e);if(0===s.length)return a;for(const t of s){const e=await t.dataset();a.push(e)}return a}}async function getWidgets(t,e,r,a=captchaOptions){let s=await getCaptchasOnce(e,r,a);if(s.length>0)return s;const{needToRefresh:i,maxRefreshTimes:n}=t;if(i&&n)for(let t=0;t<n;t++){if(!await i(e))break;if(console.log(`---------------- page refreshed ${t+1} @${new Date}`),await e.reload(),await(0,import_utils7.sleep)(5e3),s=await getCaptchasOnce(e,r,a),s.length>0)return s}else if(await(0,import_utils7.sleep)(5e3),s=await getCaptchasOnce(e,r,a),s.length>0)return s;return[]}async function sendSolution(t,e,r,a,s=captchaOptions){const{captchaType:i,widgetId:n}=r;if(e){const r={opType:"execute",captchaType:i,widgetId:n,solution:a},s=await t.evaluate(async([t,e])=>{const r=window[t];return"function"==typeof r?r(e):""},[e,r]);return"string"==typeof s?s:(logerr(`##captcha  no function window.${e}`),"failed:noMsgFunc")}{const e=`${s.rootTag} > ${s.parentTag} > ${s.captchaTag}[${captchaTypeAttr}="${i}"][${widgetIdAttr}="${n}"]`,r=await t.findElement(e);if(r){return await r.setAttribute(s.solutionAttr,a)?"ok":"failed"}return"failed:noElement"}}var amazonGetTask=function(t){checkCaptchaType(t,"amazon");const{captchaType:e,pageurl:r,sitekey:a,iv:s,context:i,challengeScript:n,captchaScript:o}=t;return{type:e,websiteURL:r,websiteKey:a,iv:s,context:i,challengeScript:n,captchaScript:o}},amazonExecutionSolution=async function(t,e,r,a){const{existing_token:s}=a;return await sendSolution(t,e,r,s)},amazonProceesor={type:"amazon",findMethods:["intercept","hunt"],scriptName:"amazon",getTask:amazonGetTask,executeSolution:amazonExecutionSolution};processors.set("amazon",amazonProceesor);var geetestGetTask=function(t){checkCaptchaType(t,"geetest");const{captchaType:e,version:r,url:a,gt:s,challenge:i,apiServer:n,captchaId:o}=t;if(!r||!["v3","v4"].includes(r))throw new Error(`## captcha invalid version ${t.version} in ${t.captchaType}`);if(!a)throw new Error(`## captcha invalid url in ${t.captchaType}`);if(!("v3"!==r||s&&i))throw new Error(`## captcha invalid gt or challenge in ${t.captchaType} v3`);if("v4"===r&&!o)throw new Error(`## captcha invalid captchaId in ${t.captchaType} v4`);const c={type:e,version:"v4"===r?4:3,websiteURL:a};return isValidStringAttr(s)&&(c.gt=s),isValidStringAttr(i)&&(c.challenge=i),isValidStringAttr(n)&&(c.geetestApiServerSubdomain=n),isValidStringAttr(o)&&(c.captchaId=o),c},geetestExecutionSolution=async function(t,e,r,a){checkCaptchaType(r,"geetest");let s="";if("v3"===r.version){const{challenge:t,validate:e,seccode:i}=a,n=i||`${e}|jordan`;if("string"!=typeof t||!t)throw new Error(`##captcha invalid challenge in ${r.captchaType}`);if("string"!=typeof e||!e)throw new Error(`##captcha invalid validate in ${r.captchaType}`);if("string"!=typeof i||!i)throw new Error(`##captcha invalid seccode in ${r.captchaType}`);s=(0,import_utils7.encodeToBase64)({geetest_challenge:t,geetest_validate:e,geetest_seccode:n})}else{if("v4"!==r.version)return"failed:invalidVersion";{const{captcha_id:t,lot_number:e,pass_token:i,gen_time:n,captcha_output:o}=a;if("string"!=typeof t||!t)throw new Error(`##captcha invalid captcha_id in ${r.captchaType}`);if("string"!=typeof e||!e)throw new Error(`##captcha invalid lot_number in ${r.captchaType}`);if("string"!=typeof i||!i)throw new Error(`##captcha invalid pass_token in ${r.captchaType}`);if("string"!=typeof n||!n)throw new Error(`##captcha invalid gen_time in ${r.captchaType}`);if("string"!=typeof o||!o)throw new Error(`##captcha invalid captcha_output in ${r.captchaType}`);s=(0,import_utils7.encodeToBase64)({captcha_id:t,lot_number:e,pass_token:i,gen_time:n,captcha_output:o})}}return await sendSolution(t,e,r,s)},geetestProceesor={type:"geetest",findMethods:["intercept"],scriptName:"geetest",getTask:geetestGetTask,executeSolution:geetestExecutionSolution};processors.set("geetest",geetestProceesor);var recaptchaGetTask=function(t,e){checkCaptchaType(t,"recaptcha");const{captchaType:r,version:a,url:s,sitekey:i,action:n,s:o,enterprise:c,invisible:l,userAgent:p,cookies:g,apiDomain:h}=t;if(!a||!["v2","v3"].includes(a))throw new Error(`## captcha invalid version ${t.version} in ${t.captchaType}`);if(!s||!i)throw new Error(`## captcha invalid url or sitekey in ${t.captchaType}`);let d=.3;"number"==typeof e?.minScore&&e.minScore>0&&e.minScore<1&&(d=e.minScore);const f={type:r,version:"v3"===a?3:2,websiteURL:s,websiteKey:i,isEnterprise:"true"===c,isInvisible:"true"===l,minScore:d};return isValidStringAttr(n)&&(f.action=n),isValidStringAttr(o)&&(f.s=o),isValidStringAttr(p)&&(f.userAgent=p),isValidStringAttr(g)&&(f.cookies=g),isValidStringAttr(h)&&(f.apiDomain=h),f},recaptchaExecutionSolution=async function(t,e,r,a){checkCaptchaType(r,"recaptcha");const{gRecaptchaResponse:s}=a;if("string"!=typeof s||!s)throw new Error(`##captcha invalid gRecaptchaResponse in ${r.captchaType}`);return await sendSolution(t,e,r,s)},recaptchaProceesor={type:"recaptcha",findMethods:["intercept","hunt"],scriptName:"recaptcha",getTask:recaptchaGetTask,executeSolution:recaptchaExecutionSolution};processors.set("recaptcha",recaptchaProceesor);var turnstileNeedToRefresh=async function(t){return 0===(await t.findElements("#cf-turnstile")).length},turnstileGetTask=function(t){checkCaptchaType(t,"turnstile");const{captchaType:e,url:r,sitekey:a,action:s,cData:i,chlPageData:n}=t,o={type:e,websiteURL:r,websiteKey:a};return isValidStringAttr(s)&&(o.action=s),isValidStringAttr(i)&&(o.cData=i),isValidStringAttr(n)&&(o.chlPageData=n),o},turnstileExecutionSolution=async function(t,e,r,a){checkCaptchaType(r,"turnstile");const{token:s}=a;if("string"!=typeof s||!s)throw new Error(`##captcha invalid token in ${r.captchaType}`);return await sendSolution(t,e,r,s)},turnstileProceesor={type:"turnstile",findMethods:["intercept","hunt"],scriptName:"turnstile",getTask:turnstileGetTask,executeSolution:turnstileExecutionSolution,needToRefresh:turnstileNeedToRefresh,maxRefreshTimes:2};async function getScripFromServer(t,e,r){try{let e=`${appConfig.cfg.urlPrefix}/script?name=${t}`;const r=await fetch(e);if(200!==r.status)return"";const a=await r.json();if(!a||0!==a.code||!a.data||"string"!=typeof a.data)return"";return Buffer.from(a.data,"base64").toString("utf8")}catch(t){return loginfo(t),""}}processors.set("turnstile",turnstileProceesor);var scriptMap=new Map;async function getScript(t){let e=scriptMap.get(t);if(e)return e;if(e=await getScripFromServer(t),"string"==typeof e&&e.length>20)return scriptMap.set(t,e),e;throw new Error(`##captcha failed to get script of ${t}`)}async function injectScripts(t,e){let r=!0;if(!Array.isArray(e)||0===e.length)return r=!1,r;const a=[];for(const s of e){const e=processors.get(s);if(!e)continue;if(!e.findMethods.includes("intercept")||!e.scriptName)continue;if(a.includes(e.scriptName))continue;const i=await getScript(e.scriptName);if(!i){r=!1;break}a.push(e.scriptName);await t.addPreloadScript(i)||(r=!1)}return t.setCloseWhenFree(!0),r}async function solveTokenCaptcha(t,e,r,a){const s={code:0,captchaType:e,taskResults:[]};console.log(`######################### start to sovle ${e} @${new Date}: `,a);let i="ok";try{const r=processors.get(e);if(!r)return logerr(`##captcha cannot process ${e}`),s.code=1001,s;const{getTask:n,executeSolution:o}=r,c=await getWidgets(r,t,"");if(console.log("--------------- captchaWidgets: ",c),0===c.length)return s;const l=c[0];e!==l.captchaType&&logerr(`##captcha different captchaType ${e} ${l.captchaType}`);const p=n(l,a);console.log("--------------- task: ",p);const{solution:g,errorCode:h,taskId:d,createTime:f,endTime:u}=await executeTask(p);if(s.taskResults.push({errorCode:h,taskId:d,elapse:u-f}),g&&(i=await o(t,"",l,g),i.startsWith("ok")||(s.code=1003,s.message=i),i)){const e="button[type='submit']",r=await t.findElement(e);r&&console.log("@@@@@@@@@@@@@@@ click: ",await r.click())}return console.log(`######################### end to sovle ${e} @${new Date}: `,s),s}catch(t){return logerr(t),s.code=1010,s.message=t.message,console.log(`######################### end to sovle ${e} @${new Date}: `,s),s}}var import_node_fs=__toESM(require("fs"),1),import_utils8=require("@letsscrapedata/utils"),processors2=new Map;async function getTextOfElement(t,e,r){if("pageUrl"===r)return t.url();const a=await t.findElement(e);return a?"textContent"===r?await a.textContent():await a.attribute(r):""}async function getBufferOfImage(t,e,r,a){const s=await t.findElement(e);if(!s)return null;try{const e=await s.attribute("src");if(e.toLowerCase().includes(".gif")){const s=t.url(),i=import_utils8.LsdUrl.getAbsoulteUrl(s,e),n=await lsdDownload(i,r?.proxyUrl);return"number"==typeof a&&import_node_fs.default.writeFileSync(`./captchaImage${a}.gif`,n),n}{"lazy"===await s.attribute("loading")&&(await s.scrollIntoView(),await(0,import_utils8.sleep)(2e3));const t=await s.screenshot();return"number"==typeof a&&import_node_fs.default.writeFileSync(`./captchaImage${a}.png`,t),t}}catch(t){return logerr(t),null}}var textGetTask=async function(t,e,r=null){const{type:a,imageLocs:s,case:i,commentLoc:n,commentAttr:o,inputLoc:c,enter:l,submitLoc:p,checkLoc:g,lang:h,math:d,maxLength:f,minLength:u,question:m,space:y,textType:C}=e;if(!Array.isArray(s)||0===s.length)throw new Error("invalidParas");if(!l&&!p)throw new Error("invalidParas");if(!(!m||n&&o))throw new Error("invalidParas");if(!g)throw new Error("invalidParas");const w=[];for(const[e,a]of s.entries()){const s=await getBufferOfImage(t,a,r,e);if(!s)throw new Error("noRequiredElement:imageLoc");w.push(s)}if(!await t.findElement(c))throw new Error("noRequiredElement:inputLoc");let k="";if(n&&(k=await getTextOfElement(t,n,o),!k))throw new Error("noRequiredElement:commentLoc");if(p){if(!await t.findElement(p))throw new Error("noRequiredElement:submitLoc")}return{type:a,images:w,case:i,comment:k,lang:h,math:d,maxLength:f,minLength:u,question:m,space:y,textType:C,websiteUrl:t.url()}},textExecuteSolution=async function(t,e,r){const{text:a}=r,{inputLoc:s,enter:i,submitLoc:n,checkLoc:o,checkAttr:c,failedStr:l}=e,p=await t.findElement(s);if(!p)throw new Error("noRequiredElement");let g=!1;if(i){if(g=await p.input(a,{enter:i}),!g)return"failed:input"}else{const e=await t.findElement(n);if(!e)throw new Error("noRequiredElement");if(g=await p.input(a),!g)return"failed:input";if(g=await e.click(),!g)return"failed:submit"}await(0,import_utils8.sleep)(1e3);const h=await t.findElement(o);if(!h)return"ok";if(!c)return"failed:checkFailed-noCheckElement";const d="textContent"===c?await h.textContent():await h.attribute(c);return!l&&l.length>0?`failed:checkFailed-${d}`:d.includes(l)?`failed:checkFailed-${l}`:"ok"},textProcessor={type:"text",getTask:textGetTask,executeSolution:textExecuteSolution};processors2.set("text",textProcessor);var coordinateGetTask=async function(t,e,r=null){const{type:a,imageLocs:s,commentLoc:i,commentAttr:n,lang:o,maxClicks:c,minClicks:l}=e;if(!Array.isArray(s)||0===s.length)throw new Error("invalidParas");if(l<0||c>0&&l>=c)throw new Error("invalidParas");const p=[];for(const[e,a]of s.entries()){const s=await getBufferOfImage(t,a,r,e);if(!s)throw new Error("noRequiredElement:imageLoc");p.push(s)}let g="";if(i&&(g=await getTextOfElement(t,i,n),!g))throw new Error("noRequiredElement:commentLoc");return{type:a,images:p,comment:g,lang:o,maxClicks:c,minClicks:l}},coordinateExecuteSolution=async function(t,e,r){const{coordinates:a}=r,{minClicks:s,maxClicks:i,imageLocs:n,submitLoc:o,checkLoc:c,checkAttr:l,failedStr:p}=e;if(!Array.isArray(a)||a.length<s||i>0&&a.length>i)return"failed:invalidSolution";let g=!1;const h=await t.findElement(n[0]);if(!h)throw new Error("noRequiredElement");const d=await h.boundingBox();if(!d)return"failed:click";const{x:f,y:u}=d;for(const{x:e,y:r}of a){if(g=await t.mouseClick(f+e,u+r),!g)return"failed:click";await(0,import_utils8.sleep)(200)}if(await(0,import_utils8.sleep)(1e3),o){const e=await t.findElement(o);if(!e)throw new Error("noRequiredElement");if(g=await e.click(),!g)return"failed:submit";await(0,import_utils8.sleep)(1e3)}const m=await t.findElement(c);if(!m)return"ok";if(!l)return"failed:checkFailed-noCheckElement";const y="textContent"===l?await m.textContent():await m.attribute(l);return!p&&p.length>0?`failed:checkFailed-${y}`:y.includes(p)?`failed:checkFailed-${p}`:"ok"},coordinateProcessor={type:"text",getTask:coordinateGetTask,executeSolution:coordinateExecuteSolution};async function solveRecognitionCaptcha(t,e,r,a=1){const s=e.type;console.log(`######################### start to sovle ${s} @${new Date}: `,e);const i={code:0,captchaType:s,taskResults:[]};let n="";try{const o=processors2.get(s);if(!o)return logerr(`##captcha cannot process ${s}`),i.code=1001,i;const{getTask:c,executeSolution:l}=o;for(let s=1;s<=a;s++){const a=await c(t,e,r);console.log("--------------- task: ",a);const{solution:s,errorCode:o,taskId:p,createTime:g,endTime:h}=await executeTask(a),d={errorCode:o,taskId:p,elapse:h-g};if(i.taskResults.push(d),s){if(n=await l(t,e,s),n.startsWith("ok"))return i;if(n.startsWith("failed:checkFailed")){d.errorCode="ERROR_CHECK_FAILED";continue}d.errorCode="ERROR_SOLUTION_FAILED";break}}return i.code=1003,i.message=n,console.log(`######################### end to sovle ${s} @${new Date}: `,i),i}catch(t){logerr(t);const e=t.message;return"invalidParas"===e?i.code=1001:e.startsWith("noRequiredElement")?i.code=1004:i.code=1010,i.message=e,console.log(`######################### end to sovle ${s} @${new Date}: `,i),i}}function calcCapCredits(t){return t>0?t>>appConfig.GEN_CREDITS_BITS:0}function calcGenCredits(t){return t>0?t&appConfig.GEN_CREDITS_AND:0}processors2.set("coordinate",coordinateProcessor);var TaskScraper=class extends import_node_events.default{#D;#P;#R;#g;#l;#p;#$;#N;#L;#F;#M;#j;#c;_taskContext;#q;_paras;#U;#H;#B;#V;#W;constructor(t,e,r,a={}){super();const{template:s,actionConfigs:i,fontsConfig:n}=t,{proxy:o=null,page:c=null,browserApiContext:l=null,standaloneApiContext:p=null}=r,{performOneTask:g=appConfig.performOneTask,taskId:h=0,skipActions:d=[],extractColumns:f=[],version:u="1.0",splitStr:m="::",authInfo:y={}}=a,C=a.stateData?a.stateData:{cookies:[],localStorage:[],headers:{},userData:{}},w=a.defaultElementSource?a.defaultElementSource:s.defaultElementSource;try{this.#D=s,this.#P=i,this.#R=n,this.#g=u,this.#l=s.templateId,this.#p=h,this.#$=Array.isArray(a._captchaOptions?.captchaTypes)?a._captchaOptions.captchaTypes:[],this.#N=w,this.#F=d,this.#M=f,this.#L=m,this.#j=g,this.#c={proxy:o,page:c,browserApiContext:l,standaloneApiContext:p,stateApiContext:l||p,taskApiContext:null,popupPage:null,browserElesStack:[],cheerioPage:null,cheerioElesStack:[],loopsStack:[],continueFlag:!1,breakFlag:!1,origStateData:C,newStateData:null,execData:{},subtasks:[],credits:0,subtaskResults:[]};const t=new SysParas(this.#c,this.#l,this.#p,this.#g);this.#q={inParas:e,sysParas:t,authInfo:y,userData:Object.assign({},C.userData),responses:[]},this.#U=this.#K();const r=C.localStorage.length>0?C.localStorage[0].origin:C.cookies.length>0?`https://${C.cookies[0].domain}`:"";this.#H={responses:[],cachedRequestUrl:"",cachedRequestHeaders:{},cachedResponseUrl:"",cachedResponseHeaders:{},updatedRequestUrl:"",updatedRequestHeaders:{},updatedResponseUrl:"",updatedResponseHeaders:{},origStateUrl:r,origStateHeaders:C.headers,updatedStateUrl:"",updatedStateHeaders:{}},this.#B={break:this.#J,captcha:this.#z,click:this.#X,continue:this.#G,exit:this.#Y,extract:this.#Q,goto:this.#Z,hover:this.#tt,ifelse:this.#et,input:this.#rt,intercept:this.#at,loopdowhile:this.#st,loopfor:this.#it,loopineles:this.#nt,loopinstr:this.#ot,misc:this.__performMiscAction,scroll:this.#ct,select:this.#lt,setvar:this.#pt,subtask:this.#gt,wait:this.#ht},this.#V={normal:-1,parasinvalid:-2,accountinvalid:-41,accountlocked:-42,loginfailed:-43,loginneeded:-49,tologin:-40,accessforbidden:-61,cfginvalid:-69,captchafailed:-52,switchproxynow:-86,switchproxyfuture:-87,other:-98,tetimeout:-79},this.#W="",this._taskContext=this.#c,this._paras=this.#q,this.#R&&logdbg(this.#R)}catch(t){throw logerr("##te  Exeception in ScraperTask.constructor"),new Error(t)}}#dt(t){const e=this.#ft(t,"source");return"default"===e?this.#N:"browser"===e?"browser":"cheerio"===e?"cheerio":void(0,import_utils9.unreachable)(e)}#ut(t){this.#c.credits=t,logwarn(new Error)}#mt(t,e){"ignore"!==t&&logwarn(`##tmp  procErrname  eName ${t} scenario ${e} kId ${this.#p} tId ${this.#l} p ${this.#U}`);let r=-69;if("string"!=typeof t||!t||!e)throw this.#c.credits=r,new Error("##te  invalid paras in _procErrname");if("ignore"===t)return!0;throw void 0!==this.#V[t]?(r=this.#V[t],this.#c.credits=r,new Error(`##te  exit with errname ${t} in ${e}`)):(this.#c.credits=r,new Error(`##te  invalid errName ${t} in ${e}`))}_procErrname(t,e){return this.#mt(t,e)}#K(){if(this.#q.inParas&&"object"==typeof this.#q.inParas){const t=Array.from(Object.keys(this.#q.inParas)),e=t.filter(t=>t.startsWith("para")),r=t.filter(t=>t.startsWith("apara")),a=t.filter(t=>t.startsWith("rpara"));let s=e.map(t=>this.#q.inParas[t]).join(this.#L);return(r.length>0||a.length>0)&&(s=`${s}@@${r.map(t=>this.#q.inParas[t]).join(this.#L)}`,a.length>0&&(s=`${s}@@${a.map(t=>this.#q.inParas[t]).join(this.#L)}`)),s}return""}#yt(t,e=!0){return this.#c.cheerioPage?this.#c.cheerioPage.load(t,e):this.#c.cheerioPage=new import_controller2.CheerioPage(t,e),!0}#Ct(t,e){if("string"!=typeof t||"string"!=typeof e||appConfig.RESERVED_PARA_NAMES.includes(t)?this.#mt("cfginvalid","setNormalVar"):"cheerioHtml"!==t&&"cheerioXml"!==t||(this.#c.cheerioElesStack.length>0&&this.#mt("cfginvalid","cheerioElesStack"),this.#yt(e,"cheerioHtml"===t)),t.startsWith("userData.")){const r=t.slice(9);0===r.length&&this.#mt("cfginvalid","invalid userData varName"),this.#q.userData[r]=e}else if(t.startsWith("inParas_")){const r=t.slice(8);0!==r.length&&void 0!==this.#q.inParas[r]||this.#mt("cfginvalid","invalid inParas varName"),this.#q.inParas[r]=e,this.#U=this.#K()}else this.#q[t]=e;return(t.startsWith("dbgdbg")||t.startsWith("cx")&&t.match(/^cx\d$/))&&logwarn(`##dbg  setvar ${t}: ${e}`),!0}#wt(t){if("string"!=typeof t)return!1;const e=t.toLowerCase();return e.startsWith("https://")||e.startsWith("http://")}#kt(t){if(t?.[appConfig.XML_ATTR_KEY]?.type)return!0;throw new Error("##te  Invalid element type config(validate)")}#_t(t,e){if("string"!=typeof t)throw new Error(`##te  Invalid templStr: ${t}`);if(t.indexOf("${")<0)return t;try{return function(t){const e=Object.keys(t),r=Object.values(t);return new Function(...e,`return \`${this}\`;`)(...r)}.call(t,e)}catch(t){return""}}#ft(t,e,r=!0,a=!0){if(!t||"string"!=typeof e)throw new Error(`##te  Invalid config or attribute, attribute: ${e}`);const s=t[appConfig.XML_ATTR_KEY][e];if(null===s)return s;if(void 0===s){if(a)throw new Error(`##te  Config has no attribute ${e}`);return""}return!r||["type","name","version"].includes(e)?s:this.#_t(s,this.#q)}_getCfgStrAttr(t,e,r=!0,a=!0){return this.#ft(t,e,r,a)}#vt(t,e){if(!t)throw new Error(`##te  Invalid action config or attribute, attribute: ${e}`);const r=t[appConfig.XML_ATTR_KEY][e];if("boolean"!=typeof r)throw loginfo(`##te  attrNames: ${JSON.stringify(t[appConfig.XML_ATTR_KEY])}, attrName: ${e}`),new Error(`##te  Element has no bool attribute ${e} or it is undefined/null`);return r}_getCfgBoolAttr(t,e){return this.#vt(t,e)}#St(t,e,r=!1){try{const a=appConfig.XML_CHILDREN_KEY,s=appConfig.XML_ATTR_KEY;let i=t[a]?t[a][e]:null;if(Array.isArray(i))return i;if(void 0===i){const t={};return"elecontent"===e?(t[s]={type:"text",multieles:!0,trim:!0,join:"\n",line:!1,top:!1},[t]):"ocrcfg"===e?(t[s]={type:"api",nlchars:"",seperator:!1,lang:"zh"},[t]):[]}if(r)throw new Error(`##te  Invalid cfgType ${e} children of parentCfg ${JSON.stringify(t)}`);return[]}catch(t){return logerr(t),[]}}_getCfgChildrenOf(t,e,r=!1){return this.#St(t,e,r)}#bt(t,e){const r=this.#St(t,e);return r.length>0?r[0]:null}#xt(t,e){const r=this.#St(t,e);if(0===r.length)throw new Error(`##te  There is no ${e} tag config`);return r[0]}#Et(){if(this.#c.page)return this.#c.page.pageInfo();throw new Error("Cannot get pageInfo from null browser page")}#Tt(){if(this.#c.page)return this.#c.page.browserContext().browser().isHeadless();throw new Error("Cannot get headless from null browser page")}#At(t,e="ceil"){if("string"!=typeof t)throw new Error(`##te  Inavlid math expr: ${t}`);if(!["ceil","round","floor"].includes(e))throw new Error(`##te  Inavlid round type: ${e}`);return Number(getTransformExpStr(t,[{type:"tonum"},{type:e}]))}#It(t){if("string"!=typeof t)throw new Error(`##te  Inavlid bool expr: ${t}`);return Boolean(Number(getTransformExpStr(t,[{type:"tobool"}])))}async#Ot(t){let e=parseInt(t);return e>0&&await(0,import_utils9.sleep)(e),!0}#Dt(t,e=!0,r=!0,a=1,s=1){const i=t?this.#vt(t,"gen"):e,n=t?this.#vt(t,"cap"):r,o=i?a:0,c=this.#D.capName&&n?s:0;return loginfo(`##credit  addTaskCredits: gen ${o}, cap ${c}`),this.#c.credits+=(c<<appConfig.GEN_CREDITS_BITS)+o,!0}#Pt(t,e){if(t){logdbg({loginFlag:this.#vt(t,"login"),captchaFlag:this.#vt(t,"captcha"),eurl:this.#ft(t,"eurl"),eloc:this.#ft(t,"eloc"),pn1:this.#ft(t,"pn1"),pv1:this.#ft(t,"pv1"),pn2:this.#ft(t,"pn2"),pv2:this.#ft(t,"pv2"),actType:e})}}async processNewPageLoginCaptcha(t,e,r,a,s=""){const i=t.pageInfo();this.#Pt(e,s);const n="popup",o=this.#ft(e,"datapage"),c=this.#vt(e,"popupsubtask");if("popup"===o||"popuporcurrent"===o){const t=this.#p>=101?appConfig.CHECK_POPUP_PAGE_TIMEOUT1:appConfig.CHECK_POPUP_PAGE_TIMEOUT2;setTimeout(()=>this.emit(n,{code:1009,message:"timeout",data:null}),t)}await r(...a);let l=null;if("popup"===o||"popuporcurrent"===o){const[t]=await(0,import_node_events2.once)(this,n);loginfo("Taskscraper received popup event"),t&&0===t.code&&t.data?l=t.data:loginfo(`##page  kId${i.taskId} newpage popup timeout`)}return"popup"!==o||l||this.#mt("normal","noExpectedPopupPage"),l?c?this.#c.popupPage=l:this.#c.page=l:await this.processLoginAndCaptchaPage(t,"current"),this.#Pt(null,""),!0}#Rt(t,e){"browser"===t&&this.#c.page||this.#mt("cfginvalid",`notBrowserElementInAction_${e}`)}async#J(t){this.#kt(t),this.#c.loopsStack.length>0?this.#c.breakFlag=!0:logerr("##te  Error: break cannot be called outside of loop")}#$t(t){const e=this.#xt(t,"element");return this.#ft(e,"loc")}#Nt(t,e=1){const r=this.#St(t,"element");return r.length<e?(this.#mt("cfginvalid","childElements"),[]):r.map(t=>this.#ft(t,"loc"))}async#Lt(t,e,r,a){const s=this.#xt(e,"image"),i=this.#Nt(s),n=this.#bt(e,"comment"),o=n?this.#$t(n):"",c=n?this.#ft(n,"attr"):"",l=this.#xt(e,"input"),p=this.#$t(l),g=this.#vt(l,"enter"),h=g?null:this.#bt(e,"submit"),d=g||!h?"":this.#$t(h);if(!g&&!h)return this.#mt("cfginvalid","actionCaptcha"),!1;const f=this.#xt(e,"check"),u=this.#$t(f),m=this.#ft(f,"attr"),y=this.#ft(f,"failedstr"),C=this.#Ft(e),{lang:w,math:k,maxlen:_,minlen:v,question:S,space:b,texttype:x}=C,E={type:"text",imageLocs:i,commentLoc:o,commentAttr:c,inputLoc:p,enter:g,submitLoc:d,checkLoc:u,checkAttr:m,failedStr:y,case:C.case,lang:w,math:k,maxLength:parseInt(_),minLength:parseInt(v),question:S,space:b,textType:x};return await solveRecognitionCaptcha(t,E,r,a)}async#Mt(t,e,r,a){const s=this.#xt(e,"image"),i=this.#Nt(s),n=this.#bt(e,"comment"),o=n?this.#$t(n):"",c=n?this.#ft(n,"attr"):"";let l="";const p=this.#bt(e,"submit");if(p&&(l=this.#$t(p),!l))return this.#mt("cfginvalid","actionCaptcha"),!1;const g=this.#xt(e,"check"),h=this.#$t(g),d=this.#ft(g,"attr"),f=this.#ft(g,"failedstr"),u=this.#Ft(e),{lang:m,maxclicks:y,minclicks:C}=u,w={type:"coordinate",imageLocs:i,commentLoc:o,commentAttr:c,submitLoc:l,checkLoc:h,checkAttr:d,failedStr:f,lang:m,maxClicks:parseInt(y),minClicks:parseInt(C)};return await solveRecognitionCaptcha(t,w,r,a)}async#z(t){const e=this.#c.page;if(!e)return this.#mt("cfginvalid","browserPageNeeded"),!1;const r=parseInt(this.#ft(t,"try")),a=this.#xt(t,"captcha"),s=this.#ft(a,"type"),i=this.#ft(t,"errname"),n=this.#c.proxy;let o;switch(s){case"amazon":case"funcaptcha":case"geetest":case"keycaptcha":case"mtcaptcha":case"turnstile":o=await solveTokenCaptcha(e,s,n);break;case"recaptcha":const t=Number(this.#ft(a,"minscore"));o=await solveTokenCaptcha(e,s,n,{minScore:t});break;case"text":case"coordinate":o=await this.#Mt(e,a,n,r);break;case"grid":case"slider":case"rotation":o=await this.#Lt(e,a,n,r);break;default:(0,import_utils9.unreachable)(s)}return 0!==o?.code&&this.#mt(i,"actionCaptcha"),!0}async#X(t){const e=this.#c.page;if(!e)return this.#mt("cfginvalid","browserPageNeeded"),!1;this.#kt(t);const r=this.#ft(t,"type"),a=this.#ft(t,"clicktype"),s=parseInt(this.#ft(t,"try")),i=this.#ft(t,"datapage"),n=this.#vt(t,"navigate"),o=this.#ft(t,"errname"),c=this.#St(t,"element",!0),{elementSource:l,elements:p}=await this.#jt(c,s);if(this.#Rt(l,"click"),0===p.length)return loginfo(`## te  ----  no element to click, actCfg: ${JSON.stringify(t).slice(0,200)}`),this.#mt(o,"actionClick"),!1;const g=p[0];try{logdbg(`##te  click text: ${await g.innerText()}`)}catch(t){logdbg(t)}e.browserContext().browser().isHeadless()||(logdbg(`##te  pageId${e.id()} bringToFront in click`),await e.bringToFront());const h=this.#ft(t,"downloadpath"),d=appConfig.cfg.baseDownloadDir,f=path4.join(d,h);if(h&&d){await(0,import_utils9.fsCheckOrCreateSubdir)(d,h,"")?await e.sendCDPMessage("Page.setDownloadBehavior",{behavior:"allow",downloadPath:f}):this.#mt("other","actionClickDwlRelDir")}if("click"===a||"evaluate"===a?await this.processNewPageLoginCaptcha(e,t,async t=>{await t.click({clickType:a})},[g],r):(logerr(`##task invalid clicktype ${a} in performClickAction`),this.#mt("cfginvalid","actionClick")),"current"===i&&n)try{await(0,import_utils9.sleep)(5e3)}catch(t){}const u=this.#ft(t,"wait");await this.#Ot(u);const m=this.#ft(t,"filename");let y="";if(f&&m){y=path4.join(f,m);let t=0;const e=300;for(t=0;t<e;t++){if(await(0,import_utils9.fileExist)(y))break;await(0,import_utils9.sleep)(1e3)}t>=e&&(logwarn(`##te  download  kId ${this.#p}: failed to download file ${y}`),this.#mt("other","actionClickDwlTimeout"))}const C=this.#ft(t,"pathvarname");return C&&this.#Ct(C,y),this.#Dt(t),!0}async#G(t){return this.#kt(t),this.#c.loopsStack.length>0?(this.#c.continueFlag=!0,!0):(logerr("##te  Error: continue cannot be called outside of loop"),!1)}async#Y(t){this.#kt(t);const e=this.#ft(t,"errname");if(!e)return!1;const r=void 0!==this.#V[e]?this.#V[e]:-69;throw this.#c.credits=r,new Error(`##te  action_throw with errname ${e}`)}#qt(t){try{const e=this.#c.page,r=e?e.url():this.#q.pageUrl?this.#q.pageUrl:"";if(!r)return logwarn("Please execute page.goto or set variable pageUrl first"),t;const a=new URL(t,r);return a?a.href:t}catch(e){return t}}#Ut(t,e=""){if(!t)return null;try{return e?new RegExp(t,e):new RegExp(t)}catch(t){return this.#mt("cfginvalid","getRegExp"),null}}async#Ht(t,e){let r="text";t&&(r=this.#ft(t,"type"));let a="step00",s="",i=!0,n=!1,o="\n",c=!1,l=!1,p=!1,g=[];try{if(e.length>0){let h=e[0];switch(r){case"text":a="step11",t&&(i=this.#vt(t,"multieles"),n=this.#vt(t,"trim"),o=this.#ft(t,"join"),c=this.#vt(t,"line"),l=this.#vt(t,"top")),a="step12";for(let t of e){if(s=await t.innerText(l),n&&(s=s.trim()),!i)break;g.push(s)}a="step13",i&&(s="stringify"===o?JSON.stringify(g):g.join(o)),c&&(s=s.replace(/[\s\r\n]+/g," ")),a="step14";break;case"attr":a="step21";const d=this.#ft(t,"attrname"),f=this.#ft(t,"attrname2"),u=this.#ft(t,"attrname3"),m=this.#ft(t,"attrname4");n=this.#vt(t,"trim");const y=this.#ft(t,"pattern"),C=this.#vt(t,"boolattr");i=this.#vt(t,"multieles"),o=this.#ft(t,"join"),c=this.#vt(t,"line"),p=this.#vt(t,"absolute"),a="step22";for(let t of e){if(C)a="step22",s=await t.hasAttribute(d)?"1":"0";else{a="step23";const e=this.#Ut(y);for(const r of[d,f,u,m]){if(!r)break;let a=await t.attribute(r);if(s="string"==typeof a?a:null===a?"":String(a),n&&(s=s.trim()),!e)break;if(s.match(e))break;s=""}p&&(s=this.#qt(s))}if(!i)break;g.push(s),a="step24"}i&&(s="stringify"===o?JSON.stringify(g):g.join(o)),c&&(s=s.replace(/[\s\r\n]+/g," ")),a="step25";break;case"innerhtml":s=await h.innerHtml();break;case"length":s=String(e.length);break;case"outerhtml":s=await h.outerHtml();break;case"pictext":s="pictext";break;case"image":s="image";break;case"query":s="query";break;default:throw new Error(`##te  Invalid content type: ${r}`)}}else"length"===r&&(s="0");return s}catch(i){throw loginfo(`##te  processBrowserElecontentCfg: tId${this.#l} ecCfg: ${JSON.stringify(t).slice(0,200)}`),loginfo(`##te  err: ${JSON.stringify(i)}`),loginfo(`##te  contentType ${r}, content ${s}, onelineFlag ${c}`),loginfo(`##te  message ${a}`),loginfo(`##te  $browserEles ${e}, length ${e.length}, e0 ${e[0]}`),new Error(i)}}#Bt(t,e){try{let r="";const a=this.#ft(t,"defaultval"),s=this.#vt(t,"usevar");let i=this.#St(t,"fun",!0).map(t=>t[appConfig.XML_ATTR_KEY]);if(s){const t=i;i=[];for(const e of t){const t=Object.assign({},e);for(const e of Object.keys(t))"string"==typeof t[e]&&t[e].includes("${")&&(t[e]=this.#_t(t[e],this.#q));i.push(t)}}return r=getTransformExpStr(e,i,a),r}catch(e){throw loginfo(`##te  processTransformCfg: tId${this.#l} tCfg: ${JSON.stringify(t).slice(0,200)}`),loginfo(`##te  err: ${JSON.stringify(e)}`),new Error(e)}}async#Vt(t){if(t){if(!this.#c.popupPage)throw new Error("No valid popup page");return this.#c.popupPage}if(this.#c.page){const t=await this.#c.page.browserContext().getPage(!0);if(t)return t;throw new Error("No new page to perform sync subtask")}throw new Error("logic error in #getSubtaskPage")}async#Wt(t){return await t.free(),!0}#Kt(t){const{proxy:e,browserApiContext:r,standaloneApiContext:a}=this.#c;return{proxy:e,page:t,browserApiContext:r,standaloneApiContext:a}}async#Jt(t){try{let e="";const r=parseInt(this.#ft(t,"tid")),a=parseInt(this.#ft(t,"minlen")),s=this.#ft(t,"parasstr");let i=this.#ft(t,"errname");const n=this.#vt(t,"popuppage");if(n&&!this.#c.popupPage){const t="ignore"===i?"cfginvalid":i;this.#mt(t,"subtaskSynMemNopage")}if(s.length>=a){const t=await this.#Vt(n),{taskData:a}=await this.#j(r,s,this.#Kt(t),"memSync");await this.#Wt(t);const{credits:i,execData:o}=a;if(n&&(this.#c.popupPage=null),logdbg(`##te  memsyncsubtask t${r} p${s} c${i} data: ${JSON.stringify(o)}`),i>=0){if(o&&1===Object.keys(o).length&&Array.isArray(Object.values(o)[0])){const t=Object.values(o)[0];if(Array.isArray(t)&&1===t.length&&1===Object.keys(t[0]).length){const r=Object.keys(t[0])[0];e=t[0][r]}else e=JSON.stringify(o)}else e=JSON.stringify(o);let t=calcGenCredits(i),r=calcCapCredits(i);this.#Dt(null,!0,!0,t,r)}}else this.#mt(i,"subtaskSynMemFailed");return e}catch(e){throw loginfo(`##te  processSyncSubtask: tId${this.#l} pCfg: ${JSON.stringify(t).slice(0,200)}`),loginfo(`##te  err: ${JSON.stringify(e)}`),new Error(e)}}#zt(t,e){const r={browserListQuery:t.browserlist?t.browerlist:"",browsers:["chrome"],operatingSystems:["windows"],devices:["desktop"],httpVersion:Number(t.httpversion)>0?t.httpversion:"2"},a=this.#St(e,"httpbrowser");if(a.length>0){const t=[];a.forEach(e=>{const r=this.#ft(e,"name");r||this.#mt("cfginvalid","actGetHttpHeaders");const a={name:r},s=parseInt(this.#ft(e,"minversion"));s>0&&(a.minVersion=s);const i=parseInt(this.#ft(e,"maxversion"));i>0&&(a.maxVersion=i);const n=this.#ft(e,"httpversion");n&&Number(n)>0&&(a.httpVersion=n),t.push(a)}),r.browsers=t}const s=this.#St(e,"httpos");if(s.length>0){const t=[];s.forEach(e=>{const r=this.#ft(e,"os");r||this.#mt("cfginvalid","actGetHttpHeaders"),t.push(r)}),r.operatingSystems=t}const i=this.#St(e,"httpdevice");if(i.length>0){const t=[];i.forEach(e=>{const r=this.#ft(e,"device");r||this.#mt("cfginvalid","actGetHttpHeaders"),t.push(r)}),r.devices=t}return r}async#Xt(t,e){try{let r="",a=[];if("element"===t){const t=this.#ft(e,"type"),s=this.#St(e,"element",!0),i=this.#xt(e,"elecontent"),n=this.#ft(e,"try",!0,!1),o=n?parseInt(n):1,{elements:c}=await this.#jt(s,o);if(a=c,0===a.length){if("length"===this.#ft(i,"type"))r="0";else if("element"===t&&e[appConfig.XML_ATTR_KEY].colname){const t=this.#ft(e,"errname");this.#mt(t,"columnElement")}}else r=await this.#Ht(i,c)}else if("templstr"===t){const t=this.#xt(e,"templstr");r=this.#ft(t,"templ")}else if("get"===t){const t=this.#xt(e,"get"),a=this.#Ft(t);let s=this.#c.page,i="page";const n={paras:this.#q},o=this.#ft(t,"type");if("screenshot"===o){const e=this.#bt(t,"element");if(e){const{elements:t}=await this.#jt([e],1);if(!(t.length>0))throw new Error("##te no elements selected in get_screenshot");i="element",r=await GetPara.getParaInElement(a,t[0],n)}}else if("httpheaders"===o){const e=a.headermethod;n.interceptionData=this.#H,"genNewByHttpOptions"===e&&(n.genHeaderOptions=this.#zt(a,t));const r=[];this.#St(t,"httpheader",!0).forEach(t=>{let e=this.#ft(t,"name");if(e=e.toLowerCase().trim(),e){const a=this.#ft(t,"source"),s=this.#ft(t,"value"),i=this.#vt(t,"delete");r.push({name:e,source:a,value:s,deleteFlag:i})}}),n.newHttpHeaders=r}else if("api"===o){n.interceptionData=this.#H,n.proxyUrl=this.#c.proxy?this.#c.proxy.proxyUrl:"";const t=a.context;let e=!1;if("browser"===t)n.apiContext=this.#c.browserApiContext,e=!0;else if("standalone"===t)n.apiContext=this.#c.standaloneApiContext,e=!0;else if("state"===t)n.apiContext=this.#c.stateApiContext,e=!0;else if("task"===t){if(!this.#c.taskApiContext){const t=this.#c.proxy?{proxy:this.#c.proxy}:{};this.#c.taskApiContext=await import_controller2.controller.newApiContext(t)}n.taskApiContext=this.#c.taskApiContext,e=!0}e&&!n.apiContext&&this.#mt("cfginvalid","getPara_nullApiContext")}else"response"===o&&(n.interceptionData=this.#H);s&&"page"===i&&["content","cookies","mhtml","pdf","screenshot","title","window"].includes(a.type)?r=await GetPara.getParaInPage(a,s,n):["api","addedtaskparas","datetime","execedtaskparas","file","httpheaders","queuedcntwithparas","queuedtaskparas","random","response","special","sysparas"].includes(a.type)?r=await GetPara.getPara(a,n):["screenshot"].includes(a.type)&&"element"===i||this.#mt("cfginvalid","getPara")}else if("subtask"===t){const t=this.#xt(e,"subtask");loginfo(`##te  to execsync subtaskCfg: ${JSON.stringify(t)}`),r=await this.#Jt(t)}else{if("ocr"!==t)throw new Error(`##te  Invalid content source type: ${t}`);{let t=this.#xt(e,"ocrcfg");const a=this.#St(e,"element",!0),{elementSource:s,elements:i}=await this.#jt(a,1,!1);if(i.length>0&&(this.#Rt(s,"ocr"),r=await this.__processBrowserOcrCfg(t,i),"number"==typeof r&&r<0))throw new Error(`##te  Error with ocr content: ${r}`)}}const s=this.#bt(e,"decodefontsvg"),i=this.#bt(e,"decodefontttf");(s||i)&&(r=await this.__decodeFontInContent(r,s,i));const n=this.#bt(e,"transform");return n&&(r=this.#Bt(n,r),null===r&&this.#mt("cfginvalid","invalidFontchars")),{content:r,elements:a}}catch(t){throw loginfo(`##te  getEleOrTemplTransform: tId${this.#l} pCfg: ${JSON.stringify(e).slice(0,200)}`),loginfo(`##te  err: ${JSON.stringify(t)}`),new Error(t)}}#Gt(t,e){let r="";if(this.#c.execData[t]&&this.#c.execData[t].length>0){let a=this.#c.execData[t].length,s=this.#c.execData[t][a-1][e];r=String("boolean"==typeof s?Number(s):s)}return r}async#Yt(t,e){const r=this.#ft(t,"valerrname"),a=this.#ft(t,"pattern");try{this.#kt(t);const s=this.#ft(t,"type"),i=this.#ft(t,"colname");let{content:n}=await this.#Xt(s,t);if(0===n.length){this.#vt(t,"completed")&&(n=this.#Gt(e,i))}if("ignore"!==r&&a){const e=this.#ft(t,"flags"),s=this.#Ut(a,e);s&&!n.match(s)&&this.#mt(r,"columnContent")}this.#vt(t,"setvar")&&this.#Ct(i,n);const o=this.#vt(t,"discarded"),c={};if(!o){let e;const r=this.#ft(t,"datatype");"number"===r?e=this.#At(n,"round"):"boolen"===r&&(e=this.#It(n)),c[i]="string"===r?n:e}return c}catch(e){throw this.#c.credits>=0&&(loginfo(`##te  processColumnCfg: tId${this.#l}, fCfg ${JSON.stringify(t).slice(0,100)}`),loginfo(`##te  err: ${JSON.stringify(e)}`)),new Error(e)}}async#Q(t){this.#kt(t);const e=this.#ft(t,"tabname");if("string"!=typeof e||0===e.length)throw new Error(`##te  Invalid table name: ${e}`);this.#c.execData[e]||(this.#c.execData[e]=[]);const r={},a=this.#St(t,"column",!0);for(let t of a){const a=this.#ft(t,"colname");if(!(this.#M.length>0)||this.#M.includes(a))try{let s=await this.#Yt(t,e);Object.assign(r,s),a.startsWith("cx")&&loginfo(`##dbg  extract: ${a}=${r[a]}`)}catch(e){throw this.#c.credits>=0&&loginfo(`##te  extract column tId${this.#l} fCfg: ${JSON.stringify(t).slice(0,200)}`),e}}return this.#c.execData[e].push(r),!0}#Qt(t){try{const e=JSON.parse(t);return e&&"object"==typeof e?e:null}catch(t){this.#mt("cfginvalid","actionGotoHeaders")}}async#Z(t){const e=this.#c.page;if(!e)return this.#mt("cfginvalid","browserPageNeeded"),!1;this.#c.browserElesStack.length>0&&this.#mt("cfginvalid","inLoopOfBrowserElements");let r="";try{this.#kt(t);const a=this.#ft(t,"type");r=this.#ft(t,"url");const s=this.#ft(t,"wait"),i=this.#vt(t,"reuse"),n=this.#vt(t,"encodeuri"),o=this.#ft(t,"headers"),c=this.#ft(t,"referer");if(n&&(r=encodeURI(r)),!this.#wt(r))throw new Error(`##te  Invalid url: ${r} when trying to open this page`);if(loginfo(`##te  goto url ${r}`),i){if(e.url().includes(r))return await this.#Ot(s),this.#Dt(t),e}if(this.#W=r,o.trim()){const t=this.#Qt(o.trim());t&&await e.setExtraHTTPHeaders(t)}return await this.processNewPageLoginCaptcha(e,t,async(t,e,r)=>{r?await t.goto(e,{waitUntil:"load",timeout:0,referer:r}):await t.goto(e,{waitUntil:"load",timeout:0})},[e,r,c],a),await this.#Ot(s),this.#Dt(t),e}catch(t){if(loginfo(`##te  kId${this.#p} goto: ${r}`),-46!==this.#c.credits&&(logerr(`##te  kId${this.#p} error credits ${this.#c.credits} in performGotoAction`),logerr(t)),this.#c.credits>=0){const e=t.message?t.message:"";if(e.includes("net::ERR_INTERNET_DISCONNECTED"))this.#ut(-71);else if(e.includes("net::ERR_NAME_NOT_RESOLVED")){await(0,import_utils9.checkDns)(appConfig.CHECK_DNS_DOMAIN)?this.#ut(-69):this.#ut(-71)}else e.includes("net::ERR_CONNECTION_TIMED_OUT")?this.#ut(-72):e.includes("net::ERR_CONNECTION_REFUSED")||e.includes("net::ERR_CONNECTION_RESET")?this.#ut(-98):e.includes("net::ERR_PROXY_CONNECTION_FAILED ")||e.includes("net::ERR_EMPTY_RESPONSE")||e.includes("net::ERR_TIMED_OUT")||e.includes("net::ERR_TUNNEL_CONNECTION_FAILED")?this.#ut(-86):e.includes("net::ERR_INVALID_AUTH_CREDENTIALS")?this.#ut(-88):(e.includes("Navigation timeout"),this.#ut(-85))}throw new Error(t)}}async#Zt(t,e=10){let r=null;for(let a=0;a<e;a++)try{return await t.hover(),!0}catch(t){loginfo(t),await(0,import_utils9.sleep)(2e3),r=t}throw r}async#tt(t){const e=this.#c.page;if(!e)return this.#mt("cfginvalid","browserPageNeeded"),!1;this.#kt(t);const r=parseInt(this.#ft(t,"try")),a=this.#ft(t,"errname");try{const s=this.#St(t,"element",!0),{elementSource:i,elements:n}=await this.#jt(s,r);if(this.#Rt(i,"hover"),0===n.length)return this.#mt(a,"actionHover"),e;this.#Tt()&&await e.bringToFront(),logdbg(`##te  pageIdx${this.#Et().pageIdx} hover`),await this.#Zt(n[0]);const o=this.#ft(t,"wait");return await this.#Ot(o),this.#Dt(t),e}catch(t){return loginfo(t),this.#mt("other","actionHover"),e}}async#te(t){this.#kt(t);const e=this.#ft(t,"type");let r=!1;if(["element","templstr"].includes(e)){let a="";a=(await this.#Xt(e,t)).content,r=this.#It(a)}else{if("else"!==e)throw new Error("##te  Invalid type(condition)");r=!0}if(r){const e=this.#St(t,"action");await this.#ee(e)}return r}async#et(t){this.#kt(t);const e=this.#St(t,"condition");for(let t of e)if(await this.#te(t))break;return!0}async#rt(t){if(!this.#c.page)return this.#mt("cfginvalid","browserPageNeeded"),!1;this.#kt(t);const e=this.#ft(t,"content"),r=this.#vt(t,"replace"),a=this.#vt(t,"enter"),s=parseInt(this.#ft(t,"try")),i=this.#ft(t,"errname"),n=this.#St(t,"element",!0);try{const{elementSource:o,elements:c}=await this.#jt(n,s);if(this.#Rt(o,"input"),0===c.length)return this.#mt(i,"actionInput"),!1;await c[0].input(e,{replace:r,enter:a});const l=this.#ft(t,"wait");return await this.#Ot(l),this.#Dt(t),!0}catch(t){return logerr(t),!1}}#Ft(t,e=!0){const r=Object.assign({},t[appConfig.XML_ATTR_KEY]);if(!r||"object"!=typeof r)throw new Error("Invalid origCfg in transformStrAttrsInCfg");if(!e)return r;for(const t of Object.keys(r))"string"==typeof r[t]&&r[t].includes("${")&&(r[t]=this.#_t(r[t],this.#q));return r}#re(t){const e=[];for(const r of t){const{type:t,method:a,url:s,resourcetype:i,postdata:n}=r,o={};if(a&&"all"!==a&&(o.methods=[a.toUpperCase()]),s&&(o.url=new RegExp(s)),i&&(o.resourceTypes=[i]),n&&(o.postData=new RegExp(n)),"abort"===t){const t={requestMatch:o,action:"abort"};e.push(t)}else{if("fufill"!==t)throw new Error(`Invalid action type ${t}`);{const{fulfill:t}=r,a={requestMatch:o,action:"fulfill",fulfill:t};e.push(a)}}}return e}#ae(t){const e=[];for(const r of t){const{type:t,method:a,url:s,resourcetype:i,postdata:n,minsize:o,maxsize:c}=r,l={};a&&"all"!==a&&(l.methods=[a.toUpperCase()]),s&&(l.url=new RegExp(s)),i&&(l.resourceTypes=[i]),n&&(l.postData=new RegExp(n));const p={},g=parseInt(o);g>0&&(p.minLength=g);const h=parseInt(c);if(h>0&&(p.maxLength=h),"cache"===t){const{tabname:t,contenttype:a,responses:s,requestheaders:i,responseheaders:n}=r,o={requestMatch:l,responseMatch:p,handler:responseCacheCb,handlerOptions:{execData:this.#c.execData,contentType:a,tabname:t,responseFlag:s,requestHeadersFlag:i,responseHeadersFlag:n,interceptionData:this.#H}};e.push(o)}else{if("save"!==t)throw new Error(`Invalid response type ${t}`);{const{basedir:t,encoding:a,extname:s,hashmethod:i,hostnameparts:n,params:o,pathtype:c,tabname:g,validextnames:h}=r,d={requestMatch:l,responseMatch:p,handler:responseSaveCb,handlerOptions:{execData:this.#c.execData,baseDir:t,pathType:c,hashMethod:i,hostnameParts:parseInt(n),extname:s,validExtnames:h,params:o,encoding:a,tabname:g}};e.push(d)}}}return e}async#at(t){const e=this.#c.page;if(!e)return this.#mt("cfginvalid","browserPageNeeded"),!1;this.#kt(t);const r=this.#ft(t,"subtype");let a=this.#St(t,"request"),s=this.#St(t,"response");try{if(a=a.map(t=>this.#Ft(t)),s=s.map(t=>this.#Ft(t)),"set"===r){if(a.length<=0&&s.length<=0&&this.#mt("cfginvalid","actionIntercept"),a.length>0){const t=this.#re(a);await e.setRequestInterception(t)}if(s.length>0){const t=this.#ae(s);await e.setResponseInterception(t)}}else"clear"===r&&(await e.clearRequestInterceptions(),await e.clearResponseInterceptions());return!0}catch(t){return logerr(t),!1}}async#it(t){this.#kt(t);try{const e=this.#ft(t,"errname");let r=this.#ft(t,"roundtype"),a=this.#At(this.#ft(t,"from"),r),s=this.#At(this.#ft(t,"to"),r),i=this.#At(this.#ft(t,"step"),r),n=this.#ft(t,"varname"),o=this.#At(this.#ft(t,"maxloops"),r);if(o<0)return!1;let c=0;for(let e=a;e<=s&&!(o&&c++>=o);e+=i){this.#c.loopsStack.push(e),this.#Ct(n,String(e));const r=this.#St(t,"action");if(await this.#ee(r),this.#c.loopsStack.pop(),this.#c.continueFlag&&(this.#c.continueFlag=!1),this.#c.breakFlag){this.#c.breakFlag=!1;break}}return c<1&&this.#mt(e,"actionLoopfor"),!0}catch(t){throw t}}async#se(t){try{const e=this.#ft(t,"loc").trim(),r=this.#dt(t),a=this.#vt(t,"absolute"),s=this.#St(t,"iframe",!1).map(t=>this.#Ft(t)).map(t=>{const{srcprefix:e,loc:r}=t;return{src:e,selector:r}}),i=this.#c.page,n={elementSource:r,elements:[]};if("browser"===r){if(!i)throw new Error("Cannot get elements from null browser page");if(0===this.#c.browserElesStack.length)n.elements=await i.findElements(e,s);else{const t=this.#c.browserElesStack.slice(-1)[0];n.elements=await t.findElements(e,s,a)}return n}if("cheerio"===r){if(s.length>0)throw new Error("Iframe config is invalid in cheerio source");let t=this.#c.cheerioPage;if(!t){if(!i)throw new Error("Please set CheerioPage first, from which to select elememnts");const e=await i.content();e.length>10&&(this.#yt(e,!0),t=this.#c.cheerioPage)}if(!t)throw new Error("Cannot get elements from null cheerioPage");if(0===this.#c.cheerioElesStack.length)n.elements=await t.findElements(e);else{const t=this.#c.cheerioElesStack.slice(-1)[0];n.elements=await t.findElements(e,s,a)}return n}(0,import_utils9.unreachable)(r)}catch(e){throw loginfo(`##te  processElementCfg: tId${this.#l} eleCfg: ${JSON.stringify(t).slice(0,200)}`),new Error(e)}}async#jt(t,e=1,r=!0){try{if(0===t.length)throw new Error("No element tags");const a={elementSource:"browser",elements:[]};for(let s=0;s<e;s++){let e="";const i=[];for(const a of t){const{elementSource:t,elements:s}=await this.#se(a);if(e&&e!=t)throw new Error("Element sources of all element tags must be the same");if(e=t,s.length>0&&(s.forEach(t=>i.push(t)),r))break}if(e&&(a.elementSource=e),i.length>0)return a.elements=i,a;s>0&&(loginfo(`##tmp ${s} try failed in getEleOrTemplTransform`),await(0,import_utils9.sleep)(1e3))}return a}catch(e){throw loginfo(`##te  processElementCfgs: eleCfgs: ${JSON.stringify(t).slice(0,200)}`),new Error(e)}}async#nt(t){if(!this.#c.page)return this.#mt("cfginvalid","browserPageNeeded"),!1;this.#kt(t);const e=this.#ft(t,"varname"),r=e.startsWith("position"),a=this.#ft(t,"errname"),s=this.#St(t,"element",!0);let i=this.#At(this.#ft(t,"maxloops"),"round"),n=this.#At(this.#ft(t,"start"),"round"),o=this.#At(this.#ft(t,"end"),"round"),c=this.#At(this.#ft(t,"step"),"round");if(c<1&&(c=1),i<0)return!1;const{elementSource:l,elements:p}=await this.#jt(s,1),g=p.length;if(0===g)return this.#mt(a,"actionLoopineles"),!0;n<=0&&(n+=g),o<=0?o+=g:o>g&&(o=g);let h=0;for(let a=n-1;a<o&&!(i&&h>=i);a+=c){e&&this.#Ct(e,String(r?a+1:a));const s=p[a];"browser"===l?this.#c.browserElesStack.push(s):"cheerio"===l?this.#c.cheerioElesStack.push(s):(0,import_utils9.unreachable)(l),this.#c.loopsStack.push(s);const i=this.#St(t,"action");if(await this.#ee(i),this.#c.loopsStack.pop(),"browser"===l?this.#c.browserElesStack.pop():"cheerio"===l?this.#c.cheerioElesStack.pop():(0,import_utils9.unreachable)(l),h++,this.#c.continueFlag&&(this.#c.continueFlag=!1),this.#c.breakFlag){this.#c.breakFlag=!1;break}}return!0}async#ot(t){this.#kt(t);let e=this.#ft(t,"split");const r=this.#vt(t,"trim"),a=this.#ft(t,"errname");let s;try{s=new RegExp(e)}catch(t){throw new Error(`##te  Invalid regexp: ${s}`)}const i=r?this.#ft(t,"list").trim().split(s).map(t=>t.trim()).filter(t=>t.length>0):this.#ft(t,"list").split(s),n=this.#ft(t,"varname");let o=this.#At(this.#ft(t,"maxloops"),"round");if(o<0)return!1;let c=0;for(const e of i){if(o&&c++>=o)break;this.#c.loopsStack.push(e),this.#Ct(n,e);const r=this.#St(t,"action");if(await this.#ee(r),this.#c.loopsStack.pop(),this.#c.continueFlag&&(this.#c.continueFlag=!1),this.#c.breakFlag){this.#c.breakFlag=!1;break}}return c<1&&this.#mt(a,"actionLoopinstr"),!0}async#st(t){this.#kt(t);const e=this.#ft(t,"subtype"),r=this.#vt(t,"iswhile"),a=this.#ft(t,"varname"),s="element"===e&&this.#vt(t,"click"),i=(this.#vt(t,"navigate"),"element"===e&&this.#vt(t,"gen")),n="element"===e&&this.#vt(t,"cap"),o=s?this.#ft(t,"errname"):"ignore",c=s?this.#ft(t,"wait"):"-1";let l=this.#At(this.#ft(t,"maxloops"),"round");if(l<0)return!1;let{content:p,elements:g}=await this.#Xt(e,t),h=0;for(;0===h&&!r||this.#It(p);){a&&this.#Ct(a,String(h)),"element"!==e||!s||this.#F.includes("click")||this.#F.includes("operate")||(g.length<=0?this.#mt(o,"actionLoopdowhileElement"):(await g[0].click(),this.#Dt(t,i,n))),this.#c.loopsStack.push(h);const r=this.#St(t,"action");if(await this.#ee(r),this.#c.loopsStack.pop(),this.#c.continueFlag&&(this.#c.continueFlag=!1),this.#c.breakFlag){this.#c.breakFlag=!1;break}if(await this.#Ot(c),h++,l&&h>=l)break;let d=await this.#Xt(e,t);p=d.content,g=d.elements}return!0}async#ie(t){const e=this.#ft(t,"cookies"),r=this.#ft(t,"localstorage"),a=this.#ft(t,"headers"),s=this.#ft(t,"userdata"),{cookies:i,localStorage:n,userData:o}=this.#c.origStateData,c={cookies:[],localStorage:[],headers:{},userData:{}};let l;if("orig"===e)c.cookies=i;else if("new"===e)if(this.#c.page)c.cookies=await this.#c.page.cookies();else{if(!this.#c.standaloneApiContext)throw new Error(`##templ ${this.#l} has no page when getting stateData of page`);l||(l=await this.#c.standaloneApiContext.stateData()),c.cookies=l.cookies}if("orig"===r)c.localStorage=n;else if("new"===r)if(this.#c.page)c.localStorage=await this.#c.page.localStroage();else{if(!this.#c.standaloneApiContext)throw new Error(`##templ ${this.#l} has no page when getting stateData of page`);l||(l=await this.#c.standaloneApiContext.stateData()),c.localStorage=l.localStorage}const p=this.#H;p[a]&&(c.headers=Object.assign({},p[a])),"orig"===s?c.userData=o:"new"===s&&(c.userData=Object.assign({},this.#q.userData)),this.#c.newStateData=c}async _performGetstatedataInMisc(t){return await this.#ie(t)}async#ct(t){const e=this.#c.page;if(!e)return this.#mt("cfginvalid","browserPageNeeded"),!1;this.#kt(t);const r=this.#ft(t,"subtype");let a=["by","to"].includes(r)?parseInt(this.#ft(t,"height")):0;const s=["by","to"].includes(r)?this.#ft(t,"unit"):"none";let i=0,n="";"bodypct"===s?(n=await e.windowMember(["document","body","scrollHeight"]),i=parseInt(n)):"screenpct"===s?n=await e.windowMember(["screen","height"]):"windowpct"===s&&(n=await e.windowMember(["innerHeight"]));const o=n?parseInt(n):1;if(["bodypct","screenpct","windowpct"].includes(s)){if("number"!=typeof o||o<=0)throw new Error("##te  invalid unitInt in performScrollAction");a=Math.floor(a*o/100)}let c="by"===r?parseInt(this.#ft(t,"maxtimes")):0;const l="by"===r?parseInt(this.#ft(t,"interval")):0;if("by"===r){if(0===c){if(i<=0){const t=await e.windowMember(["document","body","scrollHeight"]);i=parseInt(t)}c=Math.ceil(i/a-1)}else c>200?c=200:c<0&&(c=10);for(let r=0;r<c;r++){await e.scrollBy(0,a),await(0,import_utils9.sleep)(l);const r=this.#St(t,"element");if(r&&r.length>0){const{elementSource:t,elements:e}=await this.#jt(r,1);if(this.#Rt(t,"scrollBy"),e.length>0)break}}}else if("to"===r)await e.scrollTo(0,a);else if("intoview"===r){const e=this.#St(t,"element",!0),{elementSource:r,elements:a}=await this.#jt(e,1);if(this.#Rt(r,"scrollIntoView"),a.length<=0){const e=this.#ft(t,"errname");return this.#mt(e,"actionScrollIntoview"),!1}await a[0].scrollIntoView()}return this.#Dt(t),!0}async#ne(t,e,r,a,s=1){if(t.length<=0)throw new Error("##te  No element cfg in select action");const{elementSource:i,elements:n}=await this.#jt(t,s);if(this.#Rt(i,"select"),n.length<=0)return this.#mt(a,"actionSelect"),!1;const o={type:"value"},c=r.split(",");return"index"===e?(o.type,o.indexes=c.map(t=>parseInt(t))):"label"===e?(o.type,o.labels=c):"value"===e?(o.type,o.values=c):(0,import_utils9.unreachable)(e),await n[0].select(o),!0}async#lt(t){const e=this.#c.page;if(!e)return this.#mt("cfginvalid","browserPageNeeded"),!1;e.title,this.#kt(t);const r=this.#ft(t,"selecttype"),a=this.#ft(t,"selectval"),s=parseInt(this.#ft(t,"try")),i=this.#ft(t,"errname"),n=this.#St(t,"element",!0);await this.#ne(n,r,a,i,s);const o=this.#ft(t,"wait");return await this.#Ot(o),this.#Dt(t),!0}async#pt(t){this.#kt(t);const e=this.#ft(t,"subtype"),r=e.slice(0,1).toUpperCase()+e.slice(1),a=this.#ft(t,"varname"),s=this.#ft(t,"valerrname"),i=this.#ft(t,"pattern"),n=this.#ft(t,"defaultval");let o=this.#ft(t,"path"),c="";if(["element","get","templstr","subtask","ocr"].includes(e))try{c=(await this.#Xt(e,t)).content}catch(t){loginfo(t),c=""}else if("dbquery"===e)c=await this.__processDbquery(t,n);else{if("file"!==e)throw new Error(`##te  Invalid subtupe ${e} in setvar action`);try{const e=this.#xt(t,"file");let s=this.#ft(e,"path");const i=this.#ft(e,"encoding"),n=this.#vt(t,"proxy"),o=s.toLocaleLowerCase();if(o.startsWith("http://")||o.startsWith("https://")){const t=n&&this.#c.proxy?this.#c.proxy.proxyUrl:"";c=(await lsdDownload(s,t)).toString(i)}else path4.isAbsolute(s)?appConfig.cfg.inputFileDirPart&&!path4.dirname(s).split(path4.sep).includes(appConfig.cfg.inputFileDirPart)&&this.#mt("normal",`actionSetvar${r} ${a}`):s=path4.join(appConfig.cfg.baseFileDir,s),c=fs4.readFileSync(s,{encoding:i})}catch(t){logerr(t),c=""}}if("ignore"!==s&&i){const e=this.#ft(t,"flags"),r=this.#Ut(i,e);r&&!c.match(r)&&this.#mt(s,"setvarContent")}return!c&&n&&(c=n),this.#Ct(a,c),o&&(o=path4.join(appConfig.cfg.baseFileDir,o),fs4.writeFileSync(o,c)),!0}async#gt(t){this.#kt(t);const e=this.#ft(t,"subtasks"),r=this.#vt(t,"sameasparent");if(e&&e.startsWith("[{"))try{const t=JSON.parse(e);if(Array.isArray(t)&&t.every(t=>t.tid>0&&"string"==typeof t.parasstr))for(const e of t){if(this.#c.subtasks.find(t=>t.tid===e.tid&&t.parasstr===e.parasstr)||e.minlen>0&&e.parasstr.length<e.minlen)continue;const t={tid:e.tid,parasstr:e.parasstr};e.idx>0&&(t.idx=e.idx),r&&(t.sapFlag=!0),this.#c.subtasks.push(t)}else t.length>=1&&("string"==typeof t[0].parasstr&&"number"==typeof t[0].tid||logerr(`##te invalid subtask: ${JSON.stringify(t[0])} in performSubtaskAction`))}catch(t){logerr(t)}const a=this.#St(t,"subtask");for(const t of a){const e=parseInt(this.#ft(t,"tid")),a=parseInt(this.#ft(t,"minlen")),s=this.#ft(t,"parasstr"),i=this.#ft(t,"idx").trim(),n=i.length>0&&i.match(/^[0-9]+$/)?parseInt(i):0,o=this.#ft(t,"errname"),c=this.#vt(t,"popuppage");if(c&&!this.#c.popupPage){const t="ignore"===o?"cfginvalid":o;this.#mt(t,"subtaskSyncIndepNopage")}if(s.length>=a)if(c){const t=await this.#Vt(c),r=await this.#j(e,s,this.#Kt(t),"indSync");await this.#Wt(t),this.#c.popupPage=null;const{credits:a,execData:i}=r.taskData;a<-1?this.#mt(o,"subtaskSyncIndepFailed"):(this.#c.subtaskResults.push(r),logdbg(`##te  indsyncsubtask t${e} p${s} c${a} data: ${JSON.stringify(i)}`))}else{if(!this.#c.subtasks.find(t=>t.tid===e&&t.parasstr===s)){const t={tid:e,parasstr:s};n>0&&(t.idx=n),r&&(t.sapFlag=!0),this.#c.subtasks.push(t)}}}return!0}async#ht(t){this.#kt(t);const e=this.#ft(t,"subtype"),r=this.#ft(t,"errname"),a=this.#ft(t,"wait");if("sleep"===e){const e=parseInt(this.#ft(t,"minms")),r=parseInt(this.#ft(t,"maxms")),s=(0,import_utils9.getRandomInt)(e,r);return await(0,import_utils9.sleep)(s),await this.#Ot(a),!0}const s=this.#c.page;if(!s)return this.#mt("cfginvalid","browserPageNeeded"),!1;if("element"===e){const e=this.#St(t,"element",!0);e.length<=0&&this.#mt("cfginvalid","actionWaitElement");const a=this.#ft(e[0],"loc"),i=parseInt(this.#ft(t,"timeout")),n=this.#ft(t,"state");try{await s.waitForElement(a,{timeout:i,state:n})}catch(t){logerr(t),this.#mt(r,"actionWaitElement")}}else if("navigation"===e){const e=parseInt(this.#ft(t,"timeout")),a=this.#ft(t,"waituntil"),i=this.#ft(t,"url");a||this.#mt("cfginvalid","actionWaitNavigation");try{if(i){const t=new RegExp(i);await s.waitForNavigation({url:t,timeout:e,waitUntil:a})}else await s.waitForNavigation({timeout:e,waitUntil:a})}catch(t){logerr(t),this.#mt(r,"actionWaitNavigation")}}else this.#mt("cfginvalid","actionWait");return await this.#Ot(a),!0}async#ee(t){if(!Array.isArray(t))throw new Error("##te  Ivalid action configs(actions)");for(const e of t)try{const t=this.#ft(e,"type");if(this.#F.includes(t)||this.#F.includes("operate")&&appConfig.OPERATE_ACTIONS.includes(t))continue;if(await this.#B[t].call(this,e),this.#c.continueFlag||this.#c.breakFlag)break}catch(t){throw this.#c.credits>=0&&(loginfo(`##te  tId${this.#l}, aCfg: ${JSON.stringify(e).slice(0,200)}`),loginfo(`##te  err.stack: ${t.stack}`)),t}}async#oe(){try{const t=this.#c.page;t&&this.#$.length>0&&await injectScripts(t,this.#$);const e=this.#St(this.#P,"action");await this.#ee(e),this.emit(appConfig.EVENT_TYPES.taskComplete,{code:0,message:"ok",data:null})}catch(t){this.emit(appConfig.EVENT_TYPES.taskComplete,{code:1001,message:"failed",data:t})}}#ce(){const t={};for(const[e,r]of Object.entries(this.#q))"string"==typeof e&&"string"==typeof r&&(t[e]=r);return t}#le(){const t=this.#c.page;if(t){t.on(appConfig.EVENT_TYPES.pageClose,()=>{this.emit(appConfig.EVENT_TYPES.taskComplete,{code:1005,message:"pageClosed",data:null})});const e=this.#D.maxExecutionDuration;e>0&&setTimeout(()=>this.emit(appConfig.EVENT_TYPES.taskComplete,{code:1009,message:"timeout",data:null}),1e3*e),t.pageInfo().relatedId<=0&&t.on(appConfig.EVENT_TYPES.pagePopup,async t=>{loginfo("LsdPage received popup event");const e=t;await this.processLoginAndCaptchaPage(e,"popup")})}}#pe(){const t=this.#c.page;t&&(t.removeAllListeners(appConfig.EVENT_TYPES.pagePopup),t.removeAllListeners(appConfig.EVENT_TYPES.pageClose)),this.removeAllListeners(appConfig.EVENT_TYPES.taskComplete)}async run(){const t={templateId:this.#l,parasStr:this.#U,credits:0,execData:{},subtasks:[]},e=this.#c.page;if(e){const t=this.#Et();loginfo(`##tmp  --  start to run tId${this.#l} kId${this.#p} bIdx${t.browserIdx} pIdx${t.pageIdx}`)}else loginfo(`##tmp  --  start to run tId${this.#l} kId${this.#p} bIdx0, pIdx0`);let r;try{this.#le(),this.#oe();const[a]=await(0,import_node_events2.once)(this,appConfig.EVENT_TYPES.taskComplete);if(0===a.code);else if(1001===a.code){if(a.data instanceof Error)throw a.data;logerr(a.data)}else 1005===a.code?this.#mt("other","pageClosed"):1009===a.code&&this.#mt("tetimeout","teTimeout");e&&this.#W!==e.url()&&loginfo(`##te  kId${this.#p} endUrl: ${e.url()}`);const s=this.#c.subtasks.length;s>0&&loginfo(`##te  kId${this.#p} has ${s} subtasks`);for(let t of Object.keys(this.#c.execData)){const e=this.#c.execData[t].length;loginfo(`##te  kId${this.#p} tablename ${t} has ${e}`)}return this.#pe(),loginfo(`##tmp  teSucceeded kId${this.#p} tId${this.#l}`),t.execData=this.#c.execData,t.subtasks=this.#c.subtasks,t.credits=this.#c.credits,this.#c.subtaskResults.length>0&&(r=this.#c.subtaskResults.map(t=>t.taskData)),await this.__whenTaskEndFun(),t.credits<-30&&logwarn(`##te taskId ${this.#p} credits ${t.credits}`),{taskData:t,subtaskDatas:r,newStateData:this.#c.newStateData?this.#c.newStateData:void 0}}catch(a){t.execData=this.#c.execData,t.subtasks=this.#c.subtasks,t.credits=this.#c.credits,this.#c.subtaskResults.length>0&&(r=this.#c.subtaskResults.map(t=>t.taskData));const s=this.#c.newStateData?this.#c.newStateData:void 0,i="string"==typeof a?.stack?a.stack:"unkown",n={taskId:this.#p,message:"unknown",stack:i,variables:this.#ce()};if(this.#pe(),await this.__whenTaskEndFun(),this.#c.credits>=0)logerr(`##te  Exeception in ScraperTask.run: kId ${this.#p}`),logerr(a);else{if(-1===this.#c.credits)return{taskData:t,subtaskDatas:r,newStateData:s,misc:n};if(this.#c.credits>=-30)return{taskData:t,subtaskDatas:r,newStateData:s,misc:n};if(2===this.#p&&"check"===this.#U&&e){logdbg(`##cap  loginCheckFailed capabilityId ${0} credits: ${this.#c.credits}`)}else 11===this.#p||-69===this.#c.credits||logwarn(`##te  taskExec run execption: templateId ${this.#l}, paras ${this.#U}, credits ${this.#c.credits}`)}return logwarn(`##tmp  taskExec run execption: templateId ${this.#l}, paras ${this.#U}, credits ${this.#c.credits}`),t.credits=this.#c.credits>=0?-99:this.#c.credits,{taskData:t,subtaskDatas:r,newStateData:s,misc:n}}}async __performMiscAction(t){this.#kt(t);const e=this.#St(t,"misc");if(e.length<=0)return logerr(`##templ ${this.#l} invalid template ${this.#l} in performMiscAction`),!1;for(const t of e){const e=this.#ft(t,"type");switch(e){case"delsyncdbdata":case"extractdata":this.#mt("cfginvlid","notSupported");break;case"getstatedata":await this.#ie(t);break;default:(0,import_utils9.unreachable)(e)}}return!0}async __processDbquery(t,e){return(t||e)&&this.#mt("cfginvlid","notSupported"),""}async __processBrowserOcrCfg(t,e){return(t||e)&&this.#mt("cfginvlid","notSupported"),""}async __decodeFontInContent(t,e,r){return(t||e||r)&&this.#mt("cfginvlid","notSupported"),""}async processLoginAndCaptchaPage(t,e){return!t&&!e}async __whenTaskEndFun(){return!0}},path5=__toESM(require("path"),1),fsp2=__toESM(require("fs/promises"),1),import_utils10=require("@letsscrapedata/utils");async function moveDataFiles(){try{const t=appConfig.cfg.exportsDir;if(!t||!(0,import_utils10.isDirectory)(t))return!1;const e=await(0,import_utils10.filesInDir)(t,"general","dat_");if(0===e.length)return!0;const r=(0,import_utils10.getLocalDateNumber)(),a=path5.join(t,String(r));await(0,import_utils10.fsCheckOrCreateDir)(a,!0,!0);for(const r of e){const e=path5.join(t,r),s=path5.join(a,r);await fsp2.rename(e,s)}return!0}catch(t){return logerr(t),!1}}async function writeDataIntoOneFile(t,e,r){if(e.length<=0)return!1;const a=appConfig.cfg.dataFileFormat;if("jsonl"!==a&&!await(0,import_utils10.isFile)(t)){const a=Array.from(Object.keys(e[0]));await fsp2.writeFile(t,a.join(r).concat("\n"))}const s="jsonl"===a?e.map(t=>JSON.stringify(t)).join("\n"):e.map(t=>Object.values(t).join(r)).join("\n");return await fsp2.appendFile(t,s.concat("\n")),!0}async function saveExecDataOfTask(t){try{const{exportsDir:e,dataFileFormat:r,columnSeperator:a}=appConfig.cfg;if(!(e&&(0,import_utils10.isDirectory)(e)&&r&&a))return!1;for(const[s,i]of Object.entries(t)){const t=path5.join(e,`${s}.${r}`);await writeDataIntoOneFile(t,i,a)}return!0}catch(t){return logerr(t),!1}}async function performOneTask(t,e,r,a="indAsync",s="",i=0,n=appConfig.cfg.useNickName){try{const o=await TemplateManagerInScraper.getTemplateConfig(t,s),{template:c,actionConfigs:l,fontsConfig:p,paraCfgs:g,attrsInXml:h,captchaTypes:d,datatableMap:f}=o,u={template:c,actionConfigs:l,fontsConfig:p},{skipActions:m="",extractColumns:y="",defaultParasStr:C}=h||{},w=e||(appConfig.cfg.useParasstrInXmlIfNeeded&&C?C:""),k=TaskParser.getInParas(w,g),_=1===i&&h?{skipActions:m?m.split(","):[],extractColumns:y?y.split(","):[]}:{};_._captchaOptions={captchaTypes:d};const v=new TaskScraper(u,k,r,_),S=await v.run();if(n&&(S.taskData.execData=TaskParser.convertExecData(S.taskData.execData,f)),"indSync"===a||"indAsync"===a)if(S.taskData.credits>=-1)loginfo(`Task are completed: templateId ${t}, parasStr ${e}`),await saveExecDataOfTask(S.taskData.execData);else if("indSync"===a){const r={dat_0000000000000001:[{c01:String(t),c02:e,c03:String(S.taskData.credits),c04:(0,import_utils11.getLocalDateString)()}]};await saveExecDataOfTask(r)}return S}catch(r){const s={taskId:0,message:r.message,stack:r.stack,variables:{}};if("indSync"===a){const r={dat_0000000000000001:[{c01:String(t),c02:e,c03:"-1",c04:(0,import_utils11.getLocalDateString)()}]};await saveExecDataOfTask(r)}return{taskData:{templateId:t,parasStr:e,credits:0,execData:{},subtasks:[]},subtaskDatas:[],misc:s}}}appConfig.performOneTask=performOneTask;var path7=__toESM(require("path"),1),import_utils12=require("@letsscrapedata/utils"),import_controller3=require("@letsscrapedata/controller"),BrowserManagerInScraper=class{#ge;#he;#de;#fe;#ue(t){const e=this.#ge.templateParas.find(e=>e.templateId===t);return e?.maxConncurrency?e.maxConncurrency:this.#ge.maxConcurrency}async#me(t,e){const r=await import_controller3.controller.newApiContext({proxy:e}),a="playwright"===t.browser().browserControllerType()?t.apiContext():null;this.#fe.push({proxy:e,browserContext:t,browserApiContext:a,standaloneApiContext:r,templateMap:new Map})}constructor(){const{useStealthPlugin:t,browserControllerType:e,browserType:r,lsdLaunchOptions:a,lsdConnectOptions:s,maxConcurrency:i,templateParas:n}=appConfig.cfg,o=Array.isArray(appConfig.cfg.browserConfigs)&&appConfig.cfg.browserConfigs.length>0?appConfig.cfg.browserConfigs:[{browserControllerType:e,browserType:r,proxyUrl:""}];this.#ge={useStealthPlugin:t,browserControllerType:e,browserType:r,lsdLaunchOptions:a,lsdConnectOptions:s,browserConfigs:o,maxConcurrency:i,templateParas:n},this.#he=import_controller3.controller,["playwright","puppeteer"].includes(e),this.#de=[],this.#fe=[]}async init(){const{browserControllerType:t,browserType:e,lsdLaunchOptions:r,lsdConnectOptions:a}=this.#ge;for(const s of this.#ge.browserConfigs){const{browserControllerType:i=t,browserType:n=e,browserUrl:o,proxyUrl:c=""}=s,l=this.#ye(c);if(o){const t=Object.assign({},a,{browserUrl:o}),e=await this.#he.connect(i,n,t);this.#de.push(e);for(const t of e.browserContexts())await this.#me(t,l)}else if("string"==typeof c){const t=c?l:void 0,e=Object.assign({},r,{proxy:t}),a=await this.#he.launch(i,n,e);this.#de.push(a);for(const t of a.browserContexts())await this.#me(t,l)}}return await(0,import_utils12.sleep)(2e3),!0}#ye(t,e="",r="",a=""){return{proxyUrl:t,proxyId:e||t,username:r,password:a}}async getTaskNetworkContext(t=0){if(0===this.#fe.length)throw new Error("please call init() first to launch or connect browsers.");let e;const r=(0,import_utils12.getRandomArrayIdx)(this.#fe.length);for(const a of r){const r=this.#fe[a],{proxy:s,browserContext:i,browserApiContext:n,standaloneApiContext:o,templateMap:c}=r;if(t>0&&(e=c.get(t),e||(e={templateId:t,maxConcurrency:this.#ue(t),running:0},c.set(t,e)),e.maxConcurrency<=e.running))continue;if(!i.hasFreePage())continue;const l=await i.getPage();return l?(t>0&&e&&e.running++,l.setPageInfo({relatedId:t}),{proxy:s,page:l,browserApiContext:n,standaloneApiContext:o}):null}return null}async freePage(t){const e=t.pageInfo().relatedId;if(e>0){const r=t.browserContext().id(),a=this.#fe.find(t=>t.browserContext.id()===r);if(!a)throw new Error("No browserContext has the page to free.");const s=a.templateMap.get(e);if(!s)throw new Error(`No templateMapInfo has templateId ${e} in ${r}.`);s.running--}return await t.free(),!0}},fsp3=__toESM(require("fs/promises"),1),path6=__toESM(require("path"),1),import_utils13=require("@letsscrapedata/utils"),SchedulerInScraper=class t{#Ce;#we;#ke;#_e;runningTaskSet;static#ve="\n";static#Se=!1;constructor(t){this.#Ce=new Map,this.#we=appConfig.cfg.taskDir,this.#ke=appConfig.cfg.exitWhenCompleted,this.#_e=t,this.runningTaskSet=new Set}async#be(e){try{const r=await fsp3.readFile(e,"utf8");return r.split(t.#ve)}catch(t){return[]}}static#xe=new Map;async#Ee(e,r){if(t.#xe.get(r)){for(let e=0;e<50&&(await(0,import_utils13.sleep)(200),t.#xe.get(r));e++);if(t.#xe.get(r))return!1}t.#xe.set(r,!0);try{const a=path6.join(this.#we,`templ${r}.txt`);return e.length>0?await fsp3.writeFile(a,e.join(t.#ve)):await(0,import_utils13.isFile)(a)&&await fsp3.unlink(a),t.#xe.delete(r),!0}catch(e){return logwarn(`Failed to write task queue of tempatedId ${r} into file`),t.#xe.delete(r),!1}}async#Te(){const t=appConfig.cfg.taskDir;if(!t||!(0,import_utils13.isDirectory)(t))return!1;const e=await(0,import_utils13.filesInDir)(t,"general","templ","txt");for(const r of e){const e=parseInt(r.slice(5,-4)),a=await this.#be(path6.join(t,r));if(a.length>0){loginfo(`load unfinished ${a.length} tasks of templateId ${e}`);const t=this.#Ce.get(e);t?a.forEach(e=>t.push(e)):this.#Ce.set(e,a)}}return!0}async addNewTask(t,e){const{tid:r,parasstrs:a}=t;if(r<=0||0===a.length)return!1;let s=this.#Ce.get(r);return s||(s=[],this.#Ce.set(r,s)),a.forEach(t=>{s.includes(t)||s.push(t)}),await this.#Ee(s,r),e&&await this.tryToExecuteTasks([r]),!0}async init(t=[]){appConfig.cfg.moveDataWhenStart&&await moveDataFiles(),appConfig.cfg.loadUnfinishedTasks&&await this.#Te();for(const e of t){const{tid:t,parasstrs:r}=e;this.addNewTask({tid:t,parasstrs:r},!1)}return await this.tryToExecuteTasks(),setInterval(this.tryToExecuteTasks.bind(this),6e4),!0}#Ae(t,e){return`${t}@@${e}`}async#Ie(t,e,r){const a=await performOneTask(t,e,r,"indAsync","",0,!0),{credits:s,subtasks:i}=a.taskData;let n=new Set;const o=this.#Ce.get(t);if(o){const r=o.findIndex(t=>t===e);if(r<0?logerr(`Task queue of templateId ${t} has no parasStr: ${e}`):o.splice(r,1),s>=-1){if(0===o.length&&this.#Ce.delete(t),i.length>0){n=new Set(i.map(t=>t.tid));for(const t of n){const e=i.filter(e=>e.tid===t).map(t=>t.parasstr);await this.addNewTask({tid:t,parasstrs:e},!1)}}await this.#Ee(o,t)}else o.push(e),await this.#Ee(o,t);this.runningTaskSet.delete(this.#Ae(t,e))}else logerr(`No task queue of templateId ${t}`);return r.page&&await this.#_e.freePage(r.page),appConfig.cfg.minMiliseconds>0&&await(0,import_utils13.sleep)(appConfig.cfg.minMiliseconds),await this.tryToExecuteTasks([t],1),n.size>0&&await this.tryToExecuteTasks(),!0}async tryToExecuteTasks(e=[],r=0){if(loginfo(`----    tryToExecuteTasks templateIds ${e.join(",")} : `,this.#Ce),t.#Se)return logdbg("----    another tryToExecuteTasks is running now"),!0;this.#ke&&0===this.#Ce.size&&process.exit(0),t.#Se=!0;try{let a=0;0===e.length&&(e=Array.from(this.#Ce.keys())),logdbg(`templateIds: ${e.join(",")}`);for(const s of e){const e=this.#Ce.get(s);if(!e)break;const i=e.length;for(let n=0;n<i;n++){const i=e[n],o=this.#Ae(s,i);if(this.runningTaskSet.has(o))continue;const c=await this.#_e.getTaskNetworkContext(s);if(!c){logdbg(`No page to execute ${o}`);break}if(this.runningTaskSet.add(o),this.#Ie(s,i,c),a++,r>0&&a>=r)return t.#Se=!1,!0}}return t.#Se=!1,!0}catch(e){return logdbg(e),t.#Se=!1,!1}}},import_utils14=require("@letsscrapedata/utils");async function updateScraperConfig(t){const e=appConfig.cfg,{baseDir:r,inputFileDirPart:a,useStealthPlugin:s,browserControllerType:i,browserType:n,lsdLaunchOptions:o,lsdConnectOptions:c,browserConfigs:l,urlPrefix:p,maxConcurrency:g,readCode:h,templateParas:d,totalMaxConcurrency:f,minMiliseconds:u,exitWhenCompleted:m,useParasstrInXmlIfNeeded:y,loadUnfinishedTasks:C,moveDataWhenStart:w,dataFileFormat:k,useNickName:_,columnSeperator:v}=t;if(r)e.baseDir=r;else{const t=process.cwd();e.baseDir=path7.join(t,"scraperdata")}if(await(0,import_utils14.fsCheckOrCreateDir)(e.baseDir,!0,!0),e.baseDownloadDir=path7.join(e.baseDir,"downloads"),await(0,import_utils14.fsCheckOrCreateDir)(e.baseDownloadDir,!0,!0),e.baseFileDir=path7.join(e.baseDir,"files"),await(0,import_utils14.fsCheckOrCreateDir)(e.baseFileDir,!0,!0),t.templateDir?e.templateDir=t.templateDir:e.templateDir=path7.join(e.baseDir,"template"),await(0,import_utils14.fsCheckOrCreateDir)(e.templateDir,!0,!0),e.taskDir=path7.join(e.baseDir,"task"),await(0,import_utils14.fsCheckOrCreateDir)(e.taskDir,!0,!0),e.exportsDir=path7.join(e.baseDir,"exports"),await(0,import_utils14.fsCheckOrCreateDir)(e.exportsDir,!0,!0),a&&(e.inputFileDirPart=a),i&&(e.browserControllerType=i),n&&(e.browserType=n),o&&(e.lsdLaunchOptions=o),c&&(e.lsdConnectOptions=c),l&&(e.browserConfigs=l),p&&(e.urlPrefix=p),g&&(e.maxConcurrency=g),h&&(e.readCode=h),d&&(e.templateParas=d),f&&(e.totalMaxConcurrency=f),"number"==typeof u&&(e.minMiliseconds=u),"boolean"==typeof s&&(e.useStealthPlugin=s),"boolean"==typeof m&&(e.exitWhenCompleted=m),"boolean"==typeof y&&(e.useParasstrInXmlIfNeeded=y),"boolean"==typeof C&&(e.loadUnfinishedTasks=C),"boolean"==typeof w&&(e.moveDataWhenStart=w),k&&(e.dataFileFormat=k),"boolean"==typeof _&&(e.useNickName=_),"csv"===e.dataFileFormat?e.columnSeperator=",":"tsv"===e.dataFileFormat?e.columnSeperator="\t":"txt"===e.dataFileFormat&&v&&(e.columnSeperator=v),t.captcha){const{clientKey:r}=t.captcha;"string"==typeof r&&r&&(e.captcha.clientKey=r)}return!0}async function scraper(t=[],e={}){await updateScraperConfig(e);const r=new BrowserManagerInScraper;await r.init();const a=new SchedulerInScraper(r);return await a.init(t),!0}