@humain/terminal 0.1.1 → 0.2.2

package/dist/bundle/chunks/{extract-TRW5FBY6.js → extract-CTIFQZDJ.js}

@@ -1,6 +1,5 @@
 import { createRequire as __piCreateRequire } from "node:module"; const require = __piCreateRequire(import.meta.url);
-import{require_turndown_cjs}from"./chunk-BZQ53VGE.js";import{CredentialResolutionError,DEFAULT_MODEL,activityMonitor,assertAuthFetchUrl,authFetchRedirectGuard,checkGhAvailable,extractGitHub,extractWithFirecrawl,extractWithKagi,extractWithOllama,extractWithParallel,extractWithParallelMcp,extractWithQuerit,extractWithSearch1API,extractWithTinyFish,fetchGeminiApi,fetchRemoteUrl,formatSeconds,getApiKey,getBrowserCookiesForHosts,getLastBrowserCookieDiagnostic,getUploadBase,getVersionedApiBase,getWebSearchConfigPath,hasCredentialSource,isFirecrawlAvailable,isGatewayConfigured,isGeminiAdcAvailable,isGeminiApiAvailable,isGeminiApiAvailableWithVideo,isGeminiWebAvailable,isKagiExtractAvailable,isOllamaFetchAvailable,isParallelAvailable,isPerplexityAvailable,isQueritAvailable,isSearch1APIAvailable,isTimeoutError,isTinyFishAvailable,loadFetchContentDomainPolicy,loadSsrfConfig,mapFfmpegError,pLimit,parseHTML,queryGeminiApiWithInlineData,queryGeminiApiWithVideo,queryWithCookies,readExecError,redactCredential,redactGeminiApiResponse,resizeImage,resolveCredential,searchWithPerplexity,showGhHint,trimErrorText,validateRemoteUrl}from"./chunk-2LC3S3VV.js";import"./chunk-MA4UUCYX.js";import{__commonJS,__toESM}from"./chunk-JXWJOIH4.js";var require_Readability=__commonJS({"node_modules/@mozilla/readability/Readability.js"(exports,module){function Readability2(doc,options){if(options&&options.documentElement)doc=options,options=arguments[2];else if(!doc||!doc.documentElement)throw new Error("First argument to Readability constructor should be a document object.");if(options=options||{},this._doc=doc,this._docJSDOMParser=this._doc.firstChild.__JSDOMParser__,this._articleTitle=null,this._articleByline=null,this._articleDir=null,this._articleSiteName=null,this._attempts=[],this._metadata={},this._debug=!!options.debug,this._maxElemsToParse=options.maxElemsToParse||this.DEFAULT_MAX_ELEMS_TO_PARSE,this._nbTopCandidates=options.nbTopCandidates||this.DEFAULT_N_TOP_CANDIDATES,this._charThreshold=options.charThreshold||this.DEFAULT_CHAR_THRESHOLD,this._classesToPreserve=this.CLASSES_TO_PRESERVE.concat(options.classesToPreserve||[]),this._keepClasses=!!options.keepClasses,this._serializer=options.serializer||function(el){return el.innerHTML},this._disableJSONLD=!!options.disableJSONLD,this._allowedVideoRegex=options.allowedVideoRegex||this.REGEXPS.videos,this._linkDensityModifier=options.linkDensityModifier||0,this._flags=this.FLAG_STRIP_UNLIKELYS|this.FLAG_WEIGHT_CLASSES|this.FLAG_CLEAN_CONDITIONALLY,this._debug){let logNode=function(node){if(node.nodeType==node.TEXT_NODE)return`${node.nodeName} ("${node.textContent}")`;let attrPairs=Array.from(node.attributes||[],function(attr){return`${attr.name}="${attr.value}"`}).join(" ");return`<${node.localName} ${attrPairs}>`};this.log=function(){if(typeof console<"u"){let args=Array.from(arguments,arg=>arg&&arg.nodeType==this.ELEMENT_NODE?logNode(arg):arg);args.unshift("Reader: (Readability)"),console.log(...args)}else if(typeof dump<"u"){var msg=Array.prototype.map.call(arguments,function(x){return x&&x.nodeName?logNode(x):x}).join(" ");dump("Reader: (Readability) "+msg+`
-`)}}}else this.log=function(){}}Readability2.prototype={FLAG_STRIP_UNLIKELYS:1,FLAG_WEIGHT_CLASSES:2,FLAG_CLEAN_CONDITIONALLY:4,ELEMENT_NODE:1,TEXT_NODE:3,DEFAULT_MAX_ELEMS_TO_PARSE:0,DEFAULT_N_TOP_CANDIDATES:5,DEFAULT_TAGS_TO_SCORE:"section,h2,h3,h4,h5,h6,p,td,pre".toUpperCase().split(","),DEFAULT_CHAR_THRESHOLD:500,REGEXPS:{unlikelyCandidates:/-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i,okMaybeItsACandidate:/and|article|body|column|content|main|shadow/i,positive:/article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story/i,negative:/-ad-|hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|contact|footer|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|widget/i,extraneous:/print|archive|comment|discuss|e[\-]?mail|share|reply|all|login|sign|single|utility/i,byline:/byline|author|dateline|writtenby|p-author/i,replaceFonts:/<(\/?)font[^>]*>/gi,normalize:/\s{2,}/g,videos:/\/\/(www\.)?((dailymotion|youtube|youtube-nocookie|player\.vimeo|v\.qq)\.com|(archive|upload\.wikimedia)\.org|player\.twitch\.tv)/i,shareElements:/(\b|_)(share|sharedaddy)(\b|_)/i,nextLink:/(next|weiter|continue|>([^\|]|$)|»([^\|]|$))/i,prevLink:/(prev|earl|old|new|<|«)/i,tokenize:/\W+/g,whitespace:/^\s*$/,hasContent:/\S$/,hashUrl:/^#.+/,srcsetUrl:/(\S+)(\s+[\d.]+[xw])?(\s*(?:,|$))/g,b64DataUrl:/^data:\s*([^\s;,]+)\s*;\s*base64\s*,/i,commas:/\u002C|\u060C|\uFE50|\uFE10|\uFE11|\u2E41|\u2E34|\u2E32|\uFF0C/g,jsonLdArticleTypes:/^Article|AdvertiserContentArticle|NewsArticle|AnalysisNewsArticle|AskPublicNewsArticle|BackgroundNewsArticle|OpinionNewsArticle|ReportageNewsArticle|ReviewNewsArticle|Report|SatiricalArticle|ScholarlyArticle|MedicalScholarlyArticle|SocialMediaPosting|BlogPosting|LiveBlogPosting|DiscussionForumPosting|TechArticle|APIReference$/,adWords:/^(ad(vertising|vertisement)?|pub(licité)?|werb(ung)?|广告|Реклама|Anuncio)$/iu,loadingWords:/^((loading|正在加载|Загрузка|chargement|cargando)(…|\.\.\.)?)$/iu},UNLIKELY_ROLES:["menu","menubar","complementary","navigation","alert","alertdialog","dialog"],DIV_TO_P_ELEMS:new Set(["BLOCKQUOTE","DL","DIV","IMG","OL","P","PRE","TABLE","UL"]),ALTER_TO_DIV_EXCEPTIONS:["DIV","ARTICLE","SECTION","P","OL","UL"],PRESENTATIONAL_ATTRIBUTES:["align","background","bgcolor","border","cellpadding","cellspacing","frame","hspace","rules","style","valign","vspace"],DEPRECATED_SIZE_ATTRIBUTE_ELEMS:["TABLE","TH","TD","HR","PRE"],PHRASING_ELEMS:["ABBR","AUDIO","B","BDO","BR","BUTTON","CITE","CODE","DATA","DATALIST","DFN","EM","EMBED","I","IMG","INPUT","KBD","LABEL","MARK","MATH","METER","NOSCRIPT","OBJECT","OUTPUT","PROGRESS","Q","RUBY","SAMP","SCRIPT","SELECT","SMALL","SPAN","STRONG","SUB","SUP","TEXTAREA","TIME","VAR","WBR"],CLASSES_TO_PRESERVE:["page"],HTML_ESCAPE_MAP:{lt:"<",gt:">",amp:"&",quot:'"',apos:"'"},_postProcessContent(articleContent){this._fixRelativeUris(articleContent),this._simplifyNestedElements(articleContent),this._keepClasses||this._cleanClasses(articleContent)},_removeNodes(nodeList,filterFn){if(this._docJSDOMParser&&nodeList._isLiveNodeList)throw new Error("Do not pass live node lists to _removeNodes");for(var i=nodeList.length-1;i>=0;i--){var node=nodeList[i],parentNode=node.parentNode;parentNode&&(!filterFn||filterFn.call(this,node,i,nodeList))&&parentNode.removeChild(node)}},_replaceNodeTags(nodeList,newTagName){if(this._docJSDOMParser&&nodeList._isLiveNodeList)throw new Error("Do not pass live node lists to _replaceNodeTags");for(let node of nodeList)this._setNodeTag(node,newTagName)},_forEachNode(nodeList,fn){Array.prototype.forEach.call(nodeList,fn,this)},_findNode(nodeList,fn){return Array.prototype.find.call(nodeList,fn,this)},_someNode(nodeList,fn){return Array.prototype.some.call(nodeList,fn,this)},_everyNode(nodeList,fn){return Array.prototype.every.call(nodeList,fn,this)},_getAllNodesWithTag(node,tagNames){return node.querySelectorAll?node.querySelectorAll(tagNames.join(",")):[].concat.apply([],tagNames.map(function(tag){var collection=node.getElementsByTagName(tag);return Array.isArray(collection)?collection:Array.from(collection)}))},_cleanClasses(node){var classesToPreserve=this._classesToPreserve,className=(node.getAttribute("class")||"").split(/\s+/).filter(cls=>classesToPreserve.includes(cls)).join(" ");for(className?node.setAttribute("class",className):node.removeAttribute("class"),node=node.firstElementChild;node;node=node.nextElementSibling)this._cleanClasses(node)},_isUrl(str){try{return new URL(str),!0}catch{return!1}},_fixRelativeUris(articleContent){var baseURI=this._doc.baseURI,documentURI=this._doc.documentURI;function toAbsoluteURI(uri){if(baseURI==documentURI&&uri.charAt(0)=="#")return uri;try{return new URL(uri,baseURI).href}catch{}return uri}var links=this._getAllNodesWithTag(articleContent,["a"]);this._forEachNode(links,function(link){var href=link.getAttribute("href");if(href)if(href.indexOf("javascript:")===0)if(link.childNodes.length===1&&link.childNodes[0].nodeType===this.TEXT_NODE){var text=this._doc.createTextNode(link.textContent);link.parentNode.replaceChild(text,link)}else{for(var container=this._doc.createElement("span");link.firstChild;)container.appendChild(link.firstChild);link.parentNode.replaceChild(container,link)}else link.setAttribute("href",toAbsoluteURI(href))});var medias=this._getAllNodesWithTag(articleContent,["img","picture","figure","video","audio","source"]);this._forEachNode(medias,function(media){var src=media.getAttribute("src"),poster=media.getAttribute("poster"),srcset=media.getAttribute("srcset");if(src&&media.setAttribute("src",toAbsoluteURI(src)),poster&&media.setAttribute("poster",toAbsoluteURI(poster)),srcset){var newSrcset=srcset.replace(this.REGEXPS.srcsetUrl,function(_,p1,p2,p3){return toAbsoluteURI(p1)+(p2||"")+p3});media.setAttribute("srcset",newSrcset)}})},_simplifyNestedElements(articleContent){for(var node=articleContent;node;){if(node.parentNode&&["DIV","SECTION"].includes(node.tagName)&&!(node.id&&node.id.startsWith("readability"))){if(this._isElementWithoutContent(node)){node=this._removeAndGetNext(node);continue}else if(this._hasSingleTagInsideElement(node,"DIV")||this._hasSingleTagInsideElement(node,"SECTION")){for(var child=node.children[0],i=0;i<node.attributes.length;i++)child.setAttributeNode(node.attributes[i].cloneNode());node.parentNode.replaceChild(child,node),node=child;continue}}node=this._getNextNode(node)}},_getArticleTitle(){var doc=this._doc,curTitle="",origTitle="";try{curTitle=origTitle=doc.title.trim(),typeof curTitle!="string"&&(curTitle=origTitle=this._getInnerText(doc.getElementsByTagName("title")[0]))}catch{}var titleHadHierarchicalSeparators=!1;function wordCount(str){return str.split(/\s+/).length}if(/ [\|\-\\\/>»] /.test(curTitle)){titleHadHierarchicalSeparators=/ [\\\/>»] /.test(curTitle);let allSeparators=Array.from(origTitle.matchAll(/ [\|\-\\\/>»] /gi));curTitle=origTitle.substring(0,allSeparators.pop().index),wordCount(curTitle)<3&&(curTitle=origTitle.replace(/^[^\|\-\\\/>»]*[\|\-\\\/>»]/gi,""))}else if(curTitle.includes(": ")){var headings=this._getAllNodesWithTag(doc,["h1","h2"]),trimmedTitle=curTitle.trim(),match=this._someNode(headings,function(heading){return heading.textContent.trim()===trimmedTitle});match||(curTitle=origTitle.substring(origTitle.lastIndexOf(":")+1),wordCount(curTitle)<3?curTitle=origTitle.substring(origTitle.indexOf(":")+1):wordCount(origTitle.substr(0,origTitle.indexOf(":")))>5&&(curTitle=origTitle))}else if(curTitle.length>150||curTitle.length<15){var hOnes=doc.getElementsByTagName("h1");hOnes.length===1&&(curTitle=this._getInnerText(hOnes[0]))}curTitle=curTitle.trim().replace(this.REGEXPS.normalize," ");var curTitleWordCount=wordCount(curTitle);return curTitleWordCount<=4&&(!titleHadHierarchicalSeparators||curTitleWordCount!=wordCount(origTitle.replace(/[\|\-\\\/>»]+/g,""))-1)&&(curTitle=origTitle),curTitle},_prepDocument(){var doc=this._doc;this._removeNodes(this._getAllNodesWithTag(doc,["style"])),doc.body&&this._replaceBrs(doc.body),this._replaceNodeTags(this._getAllNodesWithTag(doc,["font"]),"SPAN")},_nextNode(node){for(var next=node;next&&next.nodeType!=this.ELEMENT_NODE&&this.REGEXPS.whitespace.test(next.textContent);)next=next.nextSibling;return next},_replaceBrs(elem){this._forEachNode(this._getAllNodesWithTag(elem,["br"]),function(br){for(var next=br.nextSibling,replaced=!1;(next=this._nextNode(next))&&next.tagName=="BR";){replaced=!0;var brSibling=next.nextSibling;next.remove(),next=brSibling}if(replaced){var p=this._doc.createElement("p");for(br.parentNode.replaceChild(p,br),next=p.nextSibling;next;){if(next.tagName=="BR"){var nextElem=this._nextNode(next.nextSibling);if(nextElem&&nextElem.tagName=="BR")break}if(!this._isPhrasingContent(next))break;var sibling=next.nextSibling;p.appendChild(next),next=sibling}for(;p.lastChild&&this._isWhitespace(p.lastChild);)p.lastChild.remove();p.parentNode.tagName==="P"&&this._setNodeTag(p.parentNode,"DIV")}})},_setNodeTag(node,tag){if(this.log("_setNodeTag",node,tag),this._docJSDOMParser)return node.localName=tag.toLowerCase(),node.tagName=tag.toUpperCase(),node;for(var replacement=node.ownerDocument.createElement(tag);node.firstChild;)replacement.appendChild(node.firstChild);node.parentNode.replaceChild(replacement,node),node.readability&&(replacement.readability=node.readability);for(var i=0;i<node.attributes.length;i++)replacement.setAttributeNode(node.attributes[i].cloneNode());return replacement},_prepArticle(articleContent){this._cleanStyles(articleContent),this._markDataTables(articleContent),this._fixLazyImages(articleContent),this._cleanConditionally(articleContent,"form"),this._cleanConditionally(articleContent,"fieldset"),this._clean(articleContent,"object"),this._clean(articleContent,"embed"),this._clean(articleContent,"footer"),this._clean(articleContent,"link"),this._clean(articleContent,"aside");var shareElementThreshold=this.DEFAULT_CHAR_THRESHOLD;this._forEachNode(articleContent.children,function(topCandidate){this._cleanMatchedNodes(topCandidate,function(node,matchString){return this.REGEXPS.shareElements.test(matchString)&&node.textContent.length<shareElementThreshold})}),this._clean(articleContent,"iframe"),this._clean(articleContent,"input"),this._clean(articleContent,"textarea"),this._clean(articleContent,"select"),this._clean(articleContent,"button"),this._cleanHeaders(articleContent),this._cleanConditionally(articleContent,"table"),this._cleanConditionally(articleContent,"ul"),this._cleanConditionally(articleContent,"div"),this._replaceNodeTags(this._getAllNodesWithTag(articleContent,["h1"]),"h2"),this._removeNodes(this._getAllNodesWithTag(articleContent,["p"]),function(paragraph){var contentElementCount=this._getAllNodesWithTag(paragraph,["img","embed","object","iframe"]).length;return contentElementCount===0&&!this._getInnerText(paragraph,!1)}),this._forEachNode(this._getAllNodesWithTag(articleContent,["br"]),function(br){var next=this._nextNode(br.nextSibling);next&&next.tagName=="P"&&br.remove()}),this._forEachNode(this._getAllNodesWithTag(articleContent,["table"]),function(table){var tbody=this._hasSingleTagInsideElement(table,"TBODY")?table.firstElementChild:table;if(this._hasSingleTagInsideElement(tbody,"TR")){var row=tbody.firstElementChild;if(this._hasSingleTagInsideElement(row,"TD")){var cell=row.firstElementChild;cell=this._setNodeTag(cell,this._everyNode(cell.childNodes,this._isPhrasingContent)?"P":"DIV"),table.parentNode.replaceChild(cell,table)}}})},_initializeNode(node){switch(node.readability={contentScore:0},node.tagName){case"DIV":node.readability.contentScore+=5;break;case"PRE":case"TD":case"BLOCKQUOTE":node.readability.contentScore+=3;break;case"ADDRESS":case"OL":case"UL":case"DL":case"DD":case"DT":case"LI":case"FORM":node.readability.contentScore-=3;break;case"H1":case"H2":case"H3":case"H4":case"H5":case"H6":case"TH":node.readability.contentScore-=5;break}node.readability.contentScore+=this._getClassWeight(node)},_removeAndGetNext(node){var nextNode=this._getNextNode(node,!0);return node.remove(),nextNode},_getNextNode(node,ignoreSelfAndKids){if(!ignoreSelfAndKids&&node.firstElementChild)return node.firstElementChild;if(node.nextElementSibling)return node.nextElementSibling;do node=node.parentNode;while(node&&!node.nextElementSibling);return node&&node.nextElementSibling},_textSimilarity(textA,textB){var tokensA=textA.toLowerCase().split(this.REGEXPS.tokenize).filter(Boolean),tokensB=textB.toLowerCase().split(this.REGEXPS.tokenize).filter(Boolean);if(!tokensA.length||!tokensB.length)return 0;var uniqTokensB=tokensB.filter(token=>!tokensA.includes(token)),distanceB=uniqTokensB.join(" ").length/tokensB.join(" ").length;return 1-distanceB},_isValidByline(node,matchString){var rel=node.getAttribute("rel"),itemprop=node.getAttribute("itemprop"),bylineLength=node.textContent.trim().length;return(rel==="author"||itemprop&&itemprop.includes("author")||this.REGEXPS.byline.test(matchString))&&!!bylineLength&&bylineLength<100},_getNodeAncestors(node,maxDepth){maxDepth=maxDepth||0;for(var i=0,ancestors=[];node.parentNode&&(ancestors.push(node.parentNode),!(maxDepth&&++i===maxDepth));)node=node.parentNode;return ancestors},_grabArticle(page){this.log("**** grabArticle ****");var doc=this._doc,isPaging=page!==null;if(page=page||this._doc.body,!page)return this.log("No body found in document. Abort."),null;for(var pageCacheHtml=page.innerHTML;;){this.log("Starting grabArticle loop");var stripUnlikelyCandidates=this._flagIsActive(this.FLAG_STRIP_UNLIKELYS),elementsToScore=[],node=this._doc.documentElement;let shouldRemoveTitleHeader=!0;for(;node;){node.tagName==="HTML"&&(this._articleLang=node.getAttribute("lang"));var matchString=node.className+" "+node.id;if(!this._isProbablyVisible(node)){this.log("Removing hidden node - "+matchString),node=this._removeAndGetNext(node);continue}if(node.getAttribute("aria-modal")=="true"&&node.getAttribute("role")=="dialog"){node=this._removeAndGetNext(node);continue}if(!this._articleByline&&!this._metadata.byline&&this._isValidByline(node,matchString)){for(var endOfSearchMarkerNode=this._getNextNode(node,!0),next=this._getNextNode(node),itemPropNameNode=null;next&&next!=endOfSearchMarkerNode;){var itemprop=next.getAttribute("itemprop");if(itemprop&&itemprop.includes("name")){itemPropNameNode=next;break}else next=this._getNextNode(next)}this._articleByline=(itemPropNameNode??node).textContent.trim(),node=this._removeAndGetNext(node);continue}if(shouldRemoveTitleHeader&&this._headerDuplicatesTitle(node)){this.log("Removing header: ",node.textContent.trim(),this._articleTitle.trim()),shouldRemoveTitleHeader=!1,node=this._removeAndGetNext(node);continue}if(stripUnlikelyCandidates){if(this.REGEXPS.unlikelyCandidates.test(matchString)&&!this.REGEXPS.okMaybeItsACandidate.test(matchString)&&!this._hasAncestorTag(node,"table")&&!this._hasAncestorTag(node,"code")&&node.tagName!=="BODY"&&node.tagName!=="A"){this.log("Removing unlikely candidate - "+matchString),node=this._removeAndGetNext(node);continue}if(this.UNLIKELY_ROLES.includes(node.getAttribute("role"))){this.log("Removing content with role "+node.getAttribute("role")+" - "+matchString),node=this._removeAndGetNext(node);continue}}if((node.tagName==="DIV"||node.tagName==="SECTION"||node.tagName==="HEADER"||node.tagName==="H1"||node.tagName==="H2"||node.tagName==="H3"||node.tagName==="H4"||node.tagName==="H5"||node.tagName==="H6")&&this._isElementWithoutContent(node)){node=this._removeAndGetNext(node);continue}if(this.DEFAULT_TAGS_TO_SCORE.includes(node.tagName)&&elementsToScore.push(node),node.tagName==="DIV"){for(var p=null,childNode=node.firstChild;childNode;){var nextSibling=childNode.nextSibling;if(this._isPhrasingContent(childNode))p!==null?p.appendChild(childNode):this._isWhitespace(childNode)||(p=doc.createElement("p"),node.replaceChild(p,childNode),p.appendChild(childNode));else if(p!==null){for(;p.lastChild&&this._isWhitespace(p.lastChild);)p.lastChild.remove();p=null}childNode=nextSibling}if(this._hasSingleTagInsideElement(node,"P")&&this._getLinkDensity(node)<.25){var newNode=node.children[0];node.parentNode.replaceChild(newNode,node),node=newNode,elementsToScore.push(node)}else this._hasChildBlockElement(node)||(node=this._setNodeTag(node,"P"),elementsToScore.push(node))}node=this._getNextNode(node)}var candidates=[];this._forEachNode(elementsToScore,function(elementToScore){if(!(!elementToScore.parentNode||typeof elementToScore.parentNode.tagName>"u")){var innerText=this._getInnerText(elementToScore);if(!(innerText.length<25)){var ancestors2=this._getNodeAncestors(elementToScore,5);if(ancestors2.length!==0){var contentScore=0;contentScore+=1,contentScore+=innerText.split(this.REGEXPS.commas).length,contentScore+=Math.min(Math.floor(innerText.length/100),3),this._forEachNode(ancestors2,function(ancestor,level){if(!(!ancestor.tagName||!ancestor.parentNode||typeof ancestor.parentNode.tagName>"u")){if(typeof ancestor.readability>"u"&&(this._initializeNode(ancestor),candidates.push(ancestor)),level===0)var scoreDivider=1;else level===1?scoreDivider=2:scoreDivider=level*3;ancestor.readability.contentScore+=contentScore/scoreDivider}})}}}});for(var topCandidates=[],c=0,cl=candidates.length;c<cl;c+=1){var candidate=candidates[c],candidateScore=candidate.readability.contentScore*(1-this._getLinkDensity(candidate));candidate.readability.contentScore=candidateScore,this.log("Candidate:",candidate,"with score "+candidateScore);for(var t=0;t<this._nbTopCandidates;t++){var aTopCandidate=topCandidates[t];if(!aTopCandidate||candidateScore>aTopCandidate.readability.contentScore){topCandidates.splice(t,0,candidate),topCandidates.length>this._nbTopCandidates&&topCandidates.pop();break}}}var topCandidate=topCandidates[0]||null,neededToCreateTopCandidate=!1,parentOfTopCandidate;if(topCandidate===null||topCandidate.tagName==="BODY"){for(topCandidate=doc.createElement("DIV"),neededToCreateTopCandidate=!0;page.firstChild;)this.log("Moving child out:",page.firstChild),topCandidate.appendChild(page.firstChild);page.appendChild(topCandidate),this._initializeNode(topCandidate)}else if(topCandidate){for(var alternativeCandidateAncestors=[],i=1;i<topCandidates.length;i++)topCandidates[i].readability.contentScore/topCandidate.readability.contentScore>=.75&&alternativeCandidateAncestors.push(this._getNodeAncestors(topCandidates[i]));var MINIMUM_TOPCANDIDATES=3;if(alternativeCandidateAncestors.length>=MINIMUM_TOPCANDIDATES)for(parentOfTopCandidate=topCandidate.parentNode;parentOfTopCandidate.tagName!=="BODY";){for(var listsContainingThisAncestor=0,ancestorIndex=0;ancestorIndex<alternativeCandidateAncestors.length&&listsContainingThisAncestor<MINIMUM_TOPCANDIDATES;ancestorIndex++)listsContainingThisAncestor+=Number(alternativeCandidateAncestors[ancestorIndex].includes(parentOfTopCandidate));if(listsContainingThisAncestor>=MINIMUM_TOPCANDIDATES){topCandidate=parentOfTopCandidate;break}parentOfTopCandidate=parentOfTopCandidate.parentNode}topCandidate.readability||this._initializeNode(topCandidate),parentOfTopCandidate=topCandidate.parentNode;for(var lastScore=topCandidate.readability.contentScore,scoreThreshold=lastScore/3;parentOfTopCandidate.tagName!=="BODY";){if(!parentOfTopCandidate.readability){parentOfTopCandidate=parentOfTopCandidate.parentNode;continue}var parentScore=parentOfTopCandidate.readability.contentScore;if(parentScore<scoreThreshold)break;if(parentScore>lastScore){topCandidate=parentOfTopCandidate;break}lastScore=parentOfTopCandidate.readability.contentScore,parentOfTopCandidate=parentOfTopCandidate.parentNode}for(parentOfTopCandidate=topCandidate.parentNode;parentOfTopCandidate.tagName!="BODY"&&parentOfTopCandidate.children.length==1;)topCandidate=parentOfTopCandidate,parentOfTopCandidate=topCandidate.parentNode;topCandidate.readability||this._initializeNode(topCandidate)}var articleContent=doc.createElement("DIV");isPaging&&(articleContent.id="readability-content");var siblingScoreThreshold=Math.max(10,topCandidate.readability.contentScore*.2);parentOfTopCandidate=topCandidate.parentNode;for(var siblings=parentOfTopCandidate.children,s=0,sl=siblings.length;s<sl;s++){var sibling=siblings[s],append=!1;if(this.log("Looking at sibling node:",sibling,sibling.readability?"with score "+sibling.readability.contentScore:""),this.log("Sibling has score",sibling.readability?sibling.readability.contentScore:"Unknown"),sibling===topCandidate)append=!0;else{var contentBonus=0;if(sibling.className===topCandidate.className&&topCandidate.className!==""&&(contentBonus+=topCandidate.readability.contentScore*.2),sibling.readability&&sibling.readability.contentScore+contentBonus>=siblingScoreThreshold)append=!0;else if(sibling.nodeName==="P"){var linkDensity=this._getLinkDensity(sibling),nodeContent=this._getInnerText(sibling),nodeLength=nodeContent.length;(nodeLength>80&&linkDensity<.25||nodeLength<80&&nodeLength>0&&linkDensity===0&&nodeContent.search(/\.( |$)/)!==-1)&&(append=!0)}}append&&(this.log("Appending node:",sibling),this.ALTER_TO_DIV_EXCEPTIONS.includes(sibling.nodeName)||(this.log("Altering sibling:",sibling,"to div."),sibling=this._setNodeTag(sibling,"DIV")),articleContent.appendChild(sibling),siblings=parentOfTopCandidate.children,s-=1,sl-=1)}if(this._debug&&this.log("Article content pre-prep: "+articleContent.innerHTML),this._prepArticle(articleContent),this._debug&&this.log("Article content post-prep: "+articleContent.innerHTML),neededToCreateTopCandidate)topCandidate.id="readability-page-1",topCandidate.className="page";else{var div=doc.createElement("DIV");for(div.id="readability-page-1",div.className="page";articleContent.firstChild;)div.appendChild(articleContent.firstChild);articleContent.appendChild(div)}this._debug&&this.log("Article content after paging: "+articleContent.innerHTML);var parseSuccessful=!0,textLength=this._getInnerText(articleContent,!0).length;if(textLength<this._charThreshold)if(parseSuccessful=!1,page.innerHTML=pageCacheHtml,this._attempts.push({articleContent,textLength}),this._flagIsActive(this.FLAG_STRIP_UNLIKELYS))this._removeFlag(this.FLAG_STRIP_UNLIKELYS);else if(this._flagIsActive(this.FLAG_WEIGHT_CLASSES))this._removeFlag(this.FLAG_WEIGHT_CLASSES);else if(this._flagIsActive(this.FLAG_CLEAN_CONDITIONALLY))this._removeFlag(this.FLAG_CLEAN_CONDITIONALLY);else{if(this._attempts.sort(function(a,b){return b.textLength-a.textLength}),!this._attempts[0].textLength)return null;articleContent=this._attempts[0].articleContent,parseSuccessful=!0}if(parseSuccessful){var ancestors=[parentOfTopCandidate,topCandidate].concat(this._getNodeAncestors(parentOfTopCandidate));return this._someNode(ancestors,function(ancestor){if(!ancestor.tagName)return!1;var articleDir=ancestor.getAttribute("dir");return articleDir?(this._articleDir=articleDir,!0):!1}),articleContent}}},_unescapeHtmlEntities(str){if(!str)return str;var htmlEscapeMap=this.HTML_ESCAPE_MAP;return str.replace(/&(quot|amp|apos|lt|gt);/g,function(_,tag){return htmlEscapeMap[tag]}).replace(/&#(?:x([0-9a-f]+)|([0-9]+));/gi,function(_,hex,numStr){var num=parseInt(hex||numStr,hex?16:10);return(num==0||num>1114111||num>=55296&&num<=57343)&&(num=65533),String.fromCodePoint(num)})},_getJSONLD(doc){var scripts=this._getAllNodesWithTag(doc,["script"]),metadata;return this._forEachNode(scripts,function(jsonLdElement){if(!metadata&&jsonLdElement.getAttribute("type")==="application/ld+json")try{var content=jsonLdElement.textContent.replace(/^\s*<!\[CDATA\[|\]\]>\s*$/g,""),parsed=JSON.parse(content);if(Array.isArray(parsed)&&(parsed=parsed.find(it=>it["@type"]&&it["@type"].match(this.REGEXPS.jsonLdArticleTypes)),!parsed))return;var schemaDotOrgRegex=/^https?\:\/\/schema\.org\/?$/,matches=typeof parsed["@context"]=="string"&&parsed["@context"].match(schemaDotOrgRegex)||typeof parsed["@context"]=="object"&&typeof parsed["@context"]["@vocab"]=="string"&&parsed["@context"]["@vocab"].match(schemaDotOrgRegex);if(!matches||(!parsed["@type"]&&Array.isArray(parsed["@graph"])&&(parsed=parsed["@graph"].find(it=>(it["@type"]||"").match(this.REGEXPS.jsonLdArticleTypes))),!parsed||!parsed["@type"]||!parsed["@type"].match(this.REGEXPS.jsonLdArticleTypes)))return;if(metadata={},typeof parsed.name=="string"&&typeof parsed.headline=="string"&&parsed.name!==parsed.headline){var title=this._getArticleTitle(),nameMatches=this._textSimilarity(parsed.name,title)>.75,headlineMatches=this._textSimilarity(parsed.headline,title)>.75;headlineMatches&&!nameMatches?metadata.title=parsed.headline:metadata.title=parsed.name}else typeof parsed.name=="string"?metadata.title=parsed.name.trim():typeof parsed.headline=="string"&&(metadata.title=parsed.headline.trim());parsed.author&&(typeof parsed.author.name=="string"?metadata.byline=parsed.author.name.trim():Array.isArray(parsed.author)&&parsed.author[0]&&typeof parsed.author[0].name=="string"&&(metadata.byline=parsed.author.filter(function(author){return author&&typeof author.name=="string"}).map(function(author){return author.name.trim()}).join(", "))),typeof parsed.description=="string"&&(metadata.excerpt=parsed.description.trim()),parsed.publisher&&typeof parsed.publisher.name=="string"&&(metadata.siteName=parsed.publisher.name.trim()),typeof parsed.datePublished=="string"&&(metadata.datePublished=parsed.datePublished.trim())}catch(err){this.log(err.message)}}),metadata||{}},_getArticleMetadata(jsonld){var metadata={},values={},metaElements=this._doc.getElementsByTagName("meta"),propertyPattern=/\s*(article|dc|dcterm|og|twitter)\s*:\s*(author|creator|description|published_time|title|site_name)\s*/gi,namePattern=/^\s*(?:(dc|dcterm|og|twitter|parsely|weibo:(article|webpage))\s*[-\.:]\s*)?(author|creator|pub-date|description|title|site_name)\s*$/i;this._forEachNode(metaElements,function(element){var elementName=element.getAttribute("name"),elementProperty=element.getAttribute("property"),content=element.getAttribute("content");if(content){var matches=null,name=null;elementProperty&&(matches=elementProperty.match(propertyPattern),matches&&(name=matches[0].toLowerCase().replace(/\s/g,""),values[name]=content.trim())),!matches&&elementName&&namePattern.test(elementName)&&(name=elementName,content&&(name=name.toLowerCase().replace(/\s/g,"").replace(/\./g,":"),values[name]=content.trim()))}}),metadata.title=jsonld.title||values["dc:title"]||values["dcterm:title"]||values["og:title"]||values["weibo:article:title"]||values["weibo:webpage:title"]||values.title||values["twitter:title"]||values["parsely-title"],metadata.title||(metadata.title=this._getArticleTitle());let articleAuthor=typeof values["article:author"]=="string"&&!this._isUrl(values["article:author"])?values["article:author"]:void 0;return metadata.byline=jsonld.byline||values["dc:creator"]||values["dcterm:creator"]||values.author||values["parsely-author"]||articleAuthor,metadata.excerpt=jsonld.excerpt||values["dc:description"]||values["dcterm:description"]||values["og:description"]||values["weibo:article:description"]||values["weibo:webpage:description"]||values.description||values["twitter:description"],metadata.siteName=jsonld.siteName||values["og:site_name"],metadata.publishedTime=jsonld.datePublished||values["article:published_time"]||values["parsely-pub-date"]||null,metadata.title=this._unescapeHtmlEntities(metadata.title),metadata.byline=this._unescapeHtmlEntities(metadata.byline),metadata.excerpt=this._unescapeHtmlEntities(metadata.excerpt),metadata.siteName=this._unescapeHtmlEntities(metadata.siteName),metadata.publishedTime=this._unescapeHtmlEntities(metadata.publishedTime),metadata},_isSingleImage(node){for(;node;){if(node.tagName==="IMG")return!0;if(node.children.length!==1||node.textContent.trim()!=="")return!1;node=node.children[0]}return!1},_unwrapNoscriptImages(doc){var imgs=Array.from(doc.getElementsByTagName("img"));this._forEachNode(imgs,function(img){for(var i=0;i<img.attributes.length;i++){var attr=img.attributes[i];switch(attr.name){case"src":case"srcset":case"data-src":case"data-srcset":return}if(/\.(jpg|jpeg|png|webp)/i.test(attr.value))return}img.remove()});var noscripts=Array.from(doc.getElementsByTagName("noscript"));this._forEachNode(noscripts,function(noscript){if(this._isSingleImage(noscript)){var tmp=doc.createElement("div");tmp.innerHTML=noscript.innerHTML;var prevElement=noscript.previousElementSibling;if(prevElement&&this._isSingleImage(prevElement)){var prevImg=prevElement;prevImg.tagName!=="IMG"&&(prevImg=prevElement.getElementsByTagName("img")[0]);for(var newImg=tmp.getElementsByTagName("img")[0],i=0;i<prevImg.attributes.length;i++){var attr=prevImg.attributes[i];if(attr.value!==""&&(attr.name==="src"||attr.name==="srcset"||/\.(jpg|jpeg|png|webp)/i.test(attr.value))){if(newImg.getAttribute(attr.name)===attr.value)continue;var attrName=attr.name;newImg.hasAttribute(attrName)&&(attrName="data-old-"+attrName),newImg.setAttribute(attrName,attr.value)}}noscript.parentNode.replaceChild(tmp.firstElementChild,prevElement)}}})},_removeScripts(doc){this._removeNodes(this._getAllNodesWithTag(doc,["script","noscript"]))},_hasSingleTagInsideElement(element,tag){return element.children.length!=1||element.children[0].tagName!==tag?!1:!this._someNode(element.childNodes,function(node){return node.nodeType===this.TEXT_NODE&&this.REGEXPS.hasContent.test(node.textContent)})},_isElementWithoutContent(node){return node.nodeType===this.ELEMENT_NODE&&!node.textContent.trim().length&&(!node.children.length||node.children.length==node.getElementsByTagName("br").length+node.getElementsByTagName("hr").length)},_hasChildBlockElement(element){return this._someNode(element.childNodes,function(node){return this.DIV_TO_P_ELEMS.has(node.tagName)||this._hasChildBlockElement(node)})},_isPhrasingContent(node){return node.nodeType===this.TEXT_NODE||this.PHRASING_ELEMS.includes(node.tagName)||(node.tagName==="A"||node.tagName==="DEL"||node.tagName==="INS")&&this._everyNode(node.childNodes,this._isPhrasingContent)},_isWhitespace(node){return node.nodeType===this.TEXT_NODE&&node.textContent.trim().length===0||node.nodeType===this.ELEMENT_NODE&&node.tagName==="BR"},_getInnerText(e,normalizeSpaces2){normalizeSpaces2=typeof normalizeSpaces2>"u"?!0:normalizeSpaces2;var textContent=e.textContent.trim();return normalizeSpaces2?textContent.replace(this.REGEXPS.normalize," "):textContent},_getCharCount(e,s){return s=s||",",this._getInnerText(e).split(s).length-1},_cleanStyles(e){if(!(!e||e.tagName.toLowerCase()==="svg")){for(var i=0;i<this.PRESENTATIONAL_ATTRIBUTES.length;i++)e.removeAttribute(this.PRESENTATIONAL_ATTRIBUTES[i]);this.DEPRECATED_SIZE_ATTRIBUTE_ELEMS.includes(e.tagName)&&(e.removeAttribute("width"),e.removeAttribute("height"));for(var cur=e.firstElementChild;cur!==null;)this._cleanStyles(cur),cur=cur.nextElementSibling}},_getLinkDensity(element){var textLength=this._getInnerText(element).length;if(textLength===0)return 0;var linkLength=0;return this._forEachNode(element.getElementsByTagName("a"),function(linkNode){var href=linkNode.getAttribute("href"),coefficient=href&&this.REGEXPS.hashUrl.test(href)?.3:1;linkLength+=this._getInnerText(linkNode).length*coefficient}),linkLength/textLength},_getClassWeight(e){if(!this._flagIsActive(this.FLAG_WEIGHT_CLASSES))return 0;var weight=0;return typeof e.className=="string"&&e.className!==""&&(this.REGEXPS.negative.test(e.className)&&(weight-=25),this.REGEXPS.positive.test(e.className)&&(weight+=25)),typeof e.id=="string"&&e.id!==""&&(this.REGEXPS.negative.test(e.id)&&(weight-=25),this.REGEXPS.positive.test(e.id)&&(weight+=25)),weight},_clean(e,tag){var isEmbed=["object","embed","iframe"].includes(tag);this._removeNodes(this._getAllNodesWithTag(e,[tag]),function(element){if(isEmbed){for(var i=0;i<element.attributes.length;i++)if(this._allowedVideoRegex.test(element.attributes[i].value))return!1;if(element.tagName==="object"&&this._allowedVideoRegex.test(element.innerHTML))return!1}return!0})},_hasAncestorTag(node,tagName,maxDepth,filterFn){maxDepth=maxDepth||3,tagName=tagName.toUpperCase();for(var depth=0;node.parentNode;){if(maxDepth>0&&depth>maxDepth)return!1;if(node.parentNode.tagName===tagName&&(!filterFn||filterFn(node.parentNode)))return!0;node=node.parentNode,depth++}return!1},_getRowAndColumnCount(table){for(var rows=0,columns=0,trs=table.getElementsByTagName("tr"),i=0;i<trs.length;i++){var rowspan=trs[i].getAttribute("rowspan")||0;rowspan&&(rowspan=parseInt(rowspan,10)),rows+=rowspan||1;for(var columnsInThisRow=0,cells=trs[i].getElementsByTagName("td"),j=0;j<cells.length;j++){var colspan=cells[j].getAttribute("colspan")||0;colspan&&(colspan=parseInt(colspan,10)),columnsInThisRow+=colspan||1}columns=Math.max(columns,columnsInThisRow)}return{rows,columns}},_markDataTables(root){for(var tables=root.getElementsByTagName("table"),i=0;i<tables.length;i++){var table=tables[i],role=table.getAttribute("role");if(role=="presentation"){table._readabilityDataTable=!1;continue}var datatable=table.getAttribute("datatable");if(datatable=="0"){table._readabilityDataTable=!1;continue}var summary=table.getAttribute("summary");if(summary){table._readabilityDataTable=!0;continue}var caption=table.getElementsByTagName("caption")[0];if(caption&&caption.childNodes.length){table._readabilityDataTable=!0;continue}var dataTableDescendants=["col","colgroup","tfoot","thead","th"],descendantExists=function(tag){return!!table.getElementsByTagName(tag)[0]};if(dataTableDescendants.some(descendantExists)){this.log("Data table because found data-y descendant"),table._readabilityDataTable=!0;continue}if(table.getElementsByTagName("table")[0]){table._readabilityDataTable=!1;continue}var sizeInfo=this._getRowAndColumnCount(table);if(sizeInfo.columns==1||sizeInfo.rows==1){table._readabilityDataTable=!1;continue}if(sizeInfo.rows>=10||sizeInfo.columns>4){table._readabilityDataTable=!0;continue}table._readabilityDataTable=sizeInfo.rows*sizeInfo.columns>10}},_fixLazyImages(root){this._forEachNode(this._getAllNodesWithTag(root,["img","picture","figure"]),function(elem){if(elem.src&&this.REGEXPS.b64DataUrl.test(elem.src)){var parts=this.REGEXPS.b64DataUrl.exec(elem.src);if(parts[1]==="image/svg+xml")return;for(var srcCouldBeRemoved=!1,i=0;i<elem.attributes.length;i++){var attr=elem.attributes[i];if(attr.name!=="src"&&/\.(jpg|jpeg|png|webp)/i.test(attr.value)){srcCouldBeRemoved=!0;break}}if(srcCouldBeRemoved){var b64starts=parts[0].length,b64length=elem.src.length-b64starts;b64length<133&&elem.removeAttribute("src")}}if(!((elem.src||elem.srcset&&elem.srcset!="null")&&!elem.className.toLowerCase().includes("lazy"))){for(var j=0;j<elem.attributes.length;j++)if(attr=elem.attributes[j],!(attr.name==="src"||attr.name==="srcset"||attr.name==="alt")){var copyTo=null;if(/\.(jpg|jpeg|png|webp)\s+\d/.test(attr.value)?copyTo="srcset":/^\s*\S+\.(jpg|jpeg|png|webp)\S*\s*$/.test(attr.value)&&(copyTo="src"),copyTo){if(elem.tagName==="IMG"||elem.tagName==="PICTURE")elem.setAttribute(copyTo,attr.value);else if(elem.tagName==="FIGURE"&&!this._getAllNodesWithTag(elem,["img","picture"]).length){var img=this._doc.createElement("img");img.setAttribute(copyTo,attr.value),elem.appendChild(img)}}}}})},_getTextDensity(e,tags){var textLength=this._getInnerText(e,!0).length;if(textLength===0)return 0;var childrenLength=0,children=this._getAllNodesWithTag(e,tags);return this._forEachNode(children,child=>childrenLength+=this._getInnerText(child,!0).length),childrenLength/textLength},_cleanConditionally(e,tag){this._flagIsActive(this.FLAG_CLEAN_CONDITIONALLY)&&this._removeNodes(this._getAllNodesWithTag(e,[tag]),function(node){var isDataTable=function(t){return t._readabilityDataTable},isList=tag==="ul"||tag==="ol";if(!isList){var listLength=0,listNodes=this._getAllNodesWithTag(node,["ul","ol"]);this._forEachNode(listNodes,list=>listLength+=this._getInnerText(list).length),isList=listLength/this._getInnerText(node).length>.9}if(tag==="table"&&isDataTable(node)||this._hasAncestorTag(node,"table",-1,isDataTable)||this._hasAncestorTag(node,"code")||[...node.getElementsByTagName("table")].some(tbl=>tbl._readabilityDataTable))return!1;var weight=this._getClassWeight(node);this.log("Cleaning Conditionally",node);var contentScore=0;if(weight+contentScore<0)return!0;if(this._getCharCount(node,",")<10){for(var p=node.getElementsByTagName("p").length,img=node.getElementsByTagName("img").length,li=node.getElementsByTagName("li").length-100,input=node.getElementsByTagName("input").length,headingDensity=this._getTextDensity(node,["h1","h2","h3","h4","h5","h6"]),embedCount=0,embeds=this._getAllNodesWithTag(node,["object","embed","iframe"]),i=0;i<embeds.length;i++){for(var j=0;j<embeds[i].attributes.length;j++)if(this._allowedVideoRegex.test(embeds[i].attributes[j].value))return!1;if(embeds[i].tagName==="object"&&this._allowedVideoRegex.test(embeds[i].innerHTML))return!1;embedCount++}var innerText=this._getInnerText(node);if(this.REGEXPS.adWords.test(innerText)||this.REGEXPS.loadingWords.test(innerText))return!0;var contentLength=innerText.length,linkDensity=this._getLinkDensity(node),textishTags=["SPAN","LI","TD"].concat(Array.from(this.DIV_TO_P_ELEMS)),textDensity=this._getTextDensity(node,textishTags),isFigureChild=this._hasAncestorTag(node,"figure"),haveToRemove=(()=>{let errs=[];return!isFigureChild&&img>1&&p/img<.5&&errs.push(`Bad p to img ratio (img=${img}, p=${p})`),!isList&&li>p&&errs.push(`Too many li's outside of a list. (li=${li} > p=${p})`),input>Math.floor(p/3)&&errs.push(`Too many inputs per p. (input=${input}, p=${p})`),!isList&&!isFigureChild&&headingDensity<.9&&contentLength<25&&(img===0||img>2)&&linkDensity>0&&errs.push(`Suspiciously short. (headingDensity=${headingDensity}, img=${img}, linkDensity=${linkDensity})`),!isList&&weight<25&&linkDensity>.2+this._linkDensityModifier&&errs.push(`Low weight and a little linky. (linkDensity=${linkDensity})`),weight>=25&&linkDensity>.5+this._linkDensityModifier&&errs.push(`High weight and mostly links. (linkDensity=${linkDensity})`),(embedCount===1&&contentLength<75||embedCount>1)&&errs.push(`Suspicious embed. (embedCount=${embedCount}, contentLength=${contentLength})`),img===0&&textDensity===0&&errs.push(`No useful content. (img=${img}, textDensity=${textDensity})`),errs.length?(this.log("Checks failed",errs),!0):!1})();if(isList&&haveToRemove){for(var x=0;x<node.children.length;x++)if(node.children[x].children.length>1)return haveToRemove;let li_count=node.getElementsByTagName("li").length;if(img==li_count)return!1}return haveToRemove}return!1})},_cleanMatchedNodes(e,filter){for(var endOfSearchMarkerNode=this._getNextNode(e,!0),next=this._getNextNode(e);next&&next!=endOfSearchMarkerNode;)filter.call(this,next,next.className+" "+next.id)?next=this._removeAndGetNext(next):next=this._getNextNode(next)},_cleanHeaders(e){let headingNodes=this._getAllNodesWithTag(e,["h1","h2"]);this._removeNodes(headingNodes,function(node){let shouldRemove=this._getClassWeight(node)<0;return shouldRemove&&this.log("Removing header with low class weight:",node),shouldRemove})},_headerDuplicatesTitle(node){if(node.tagName!="H1"&&node.tagName!="H2")return!1;var heading=this._getInnerText(node,!1);return this.log("Evaluating similarity of header:",heading,this._articleTitle),this._textSimilarity(this._articleTitle,heading)>.75},_flagIsActive(flag){return(this._flags&flag)>0},_removeFlag(flag){this._flags=this._flags&~flag},_isProbablyVisible(node){return(!node.style||node.style.display!="none")&&(!node.style||node.style.visibility!="hidden")&&!node.hasAttribute("hidden")&&(!node.hasAttribute("aria-hidden")||node.getAttribute("aria-hidden")!="true"||node.className&&node.className.includes&&node.className.includes("fallback-image"))},parse(){if(this._maxElemsToParse>0){var numTags=this._doc.getElementsByTagName("*").length;if(numTags>this._maxElemsToParse)throw new Error("Aborting parsing document; "+numTags+" elements found")}this._unwrapNoscriptImages(this._doc);var jsonLd=this._disableJSONLD?{}:this._getJSONLD(this._doc);this._removeScripts(this._doc),this._prepDocument();var metadata=this._getArticleMetadata(jsonLd);this._metadata=metadata,this._articleTitle=metadata.title;var articleContent=this._grabArticle();if(!articleContent)return null;if(this.log("Grabbed: "+articleContent.innerHTML),this._postProcessContent(articleContent),!metadata.excerpt){var paragraphs=articleContent.getElementsByTagName("p");paragraphs.length&&(metadata.excerpt=paragraphs[0].textContent.trim())}var textContent=articleContent.textContent;return{title:this._articleTitle,byline:metadata.byline||this._articleByline,dir:this._articleDir,lang:this._articleLang,content:this._serializer(articleContent),textContent,length:textContent.length,excerpt:metadata.excerpt,siteName:metadata.siteName||this._articleSiteName,publishedTime:metadata.publishedTime}}};typeof module=="object"&&(module.exports=Readability2)}});var require_Readability_readerable=__commonJS({"node_modules/@mozilla/readability/Readability-readerable.js"(exports,module){var REGEXPS={unlikelyCandidates:/-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i,okMaybeItsACandidate:/and|article|body|column|content|main|shadow/i};function isNodeVisible(node){return(!node.style||node.style.display!="none")&&!node.hasAttribute("hidden")&&(!node.hasAttribute("aria-hidden")||node.getAttribute("aria-hidden")!="true"||node.className&&node.className.includes&&node.className.includes("fallback-image"))}function isProbablyReaderable(doc,options={}){typeof options=="function"&&(options={visibilityChecker:options});var defaultOptions={minScore:20,minContentLength:140,visibilityChecker:isNodeVisible};options=Object.assign(defaultOptions,options);var nodes=doc.querySelectorAll("p, pre, article"),brNodes=doc.querySelectorAll("div > br");if(brNodes.length){var set=new Set(nodes);[].forEach.call(brNodes,function(node){set.add(node.parentNode)}),nodes=Array.from(set)}var score=0;return[].some.call(nodes,function(node){if(!options.visibilityChecker(node))return!1;var matchString=node.className+" "+node.id;if(REGEXPS.unlikelyCandidates.test(matchString)&&!REGEXPS.okMaybeItsACandidate.test(matchString)||node.matches("li p"))return!1;var textContentLength=node.textContent.trim().length;return textContentLength<options.minContentLength?!1:(score+=Math.sqrt(textContentLength-options.minContentLength),score>options.minScore)})}typeof module=="object"&&(module.exports=isProbablyReaderable)}});var require_readability=__commonJS({"node_modules/@mozilla/readability/index.js"(exports,module){var Readability2=require_Readability(),isProbablyReaderable=require_Readability_readerable();module.exports={Readability:Readability2,isProbablyReaderable}}});var import_readability=__toESM(require_readability(),1);import{existsSync as existsSync8,readFileSync as readFileSync8}from"node:fs";var import_turndown=__toESM(require_turndown_cjs(),1);function extractRSCContent(html){if(!html.includes("self.__next_f.push"))return null;let chunkMap=new Map,scriptRegex=/<script>self\.__next_f\.push\(\[1,"([\s\S]*?)"\]\)<\/script>/g;for(let match of html.matchAll(scriptRegex)){let content2;try{content2=JSON.parse('"'+match[1]+'"')}catch{continue}for(let line of content2.split(`
+import{CredentialResolutionError,DEFAULT_MODEL,activityMonitor,assertAuthFetchUrl,authFetchRedirectGuard,checkGhAvailable,extractGitHub,extractWithFirecrawl,extractWithKagi,extractWithOllama,extractWithParallel,extractWithParallelMcp,extractWithQuerit,extractWithSearch1API,extractWithTinyFish,fetchGeminiApi,fetchRemoteUrl,formatSeconds,getApiKey,getBrowserCookiesForHosts,getLastBrowserCookieDiagnostic,getUploadBase,getVersionedApiBase,getWebSearchConfigPath,hasCredentialSource,isFirecrawlAvailable,isGatewayConfigured,isGeminiAdcAvailable,isGeminiApiAvailable,isGeminiApiAvailableWithVideo,isGeminiWebAvailable,isKagiExtractAvailable,isOllamaFetchAvailable,isParallelAvailable,isPerplexityAvailable,isQueritAvailable,isSearch1APIAvailable,isTimeoutError,isTinyFishAvailable,loadFetchContentDomainPolicy,loadSsrfConfig,mapFfmpegError,pLimit,queryGeminiApiWithInlineData,queryGeminiApiWithVideo,queryWithCookies,readExecError,redactCredential,redactGeminiApiResponse,resolveCredential,searchWithPerplexity,showGhHint,trimErrorText,validateRemoteUrl}from"./chunk-D6HHRSB5.js";import"./chunk-LI2YVPWP.js";import"./chunk-JXWJOIH4.js";import{existsSync as existsSync9,readFileSync as readFileSync9}from"node:fs";function extractRSCContent(html){if(!html.includes("self.__next_f.push"))return null;let chunkMap=new Map,scriptRegex=/<script>self\.__next_f\.push\(\[1,"([\s\S]*?)"\]\)<\/script>/g;for(let match of html.matchAll(scriptRegex)){let content2;try{content2=JSON.parse('"'+match[1]+'"')}catch{continue}for(let line of content2.split(`
 `)){if(!line.trim())continue;let colonIdx=line.indexOf(":");if(colonIdx<=0||colonIdx>4)continue;let id=line.slice(0,colonIdx);if(!/^[0-9a-f]+$/i.test(id))continue;let payload=line.slice(colonIdx+1);if(!payload)continue;let existing=chunkMap.get(id);(!existing||payload.length>existing.length)&&chunkMap.set(id,payload)}}if(chunkMap.size===0)return null;let title=html.match(/<title[^>]*>([^<]+)<\/title>/)?.[1]?.split("|")[0]?.trim()||"",parsedCache=new Map;function getParsedChunk(id){if(parsedCache.has(id))return parsedCache.get(id);let chunk=chunkMap.get(id);if(!chunk||!chunk.startsWith("["))return parsedCache.set(id,null),null;try{let parsed=JSON.parse(chunk);return parsedCache.set(id,parsed),parsed}catch{return parsedCache.set(id,null),null}}let visitedRefs=new Set;function extractNode(node,ctx={inTable:!1,inCode:!1}){if(node==null)return"";if(typeof node=="string"){let refMatch=node.match(/^\$L([0-9a-f]+)$/i);if(refMatch){let refId=refMatch[1];if(visitedRefs.has(refId))return"";visitedRefs.add(refId);let refNode=getParsedChunk(refId),result=refNode?extractNode(refNode,ctx):"";return visitedRefs.delete(refId),result}return!ctx.inCode&&(node==="$undefined"||node==="$"||/^\$[A-Z]/.test(node))?"":node.trim()?node:""}if(typeof node=="number")return String(node);if(typeof node=="boolean"||!Array.isArray(node))return"";if(node[0]==="$"&&typeof node[1]=="string"){let tag=node[1],props=node[3]||{};if(["script","style","svg","path","circle","link","meta","template","button","input","nav","footer","aside"].includes(tag))return"";if(tag.startsWith("$L")){let refId=tag.slice(2);if(visitedRefs.has(refId))return"";if(props.baseId&&props.children)return`## ${String(props.children)}
 
 `;visitedRefs.add(refId);let refNode=getParsedChunk(refId),result="";return refNode?result=extractNode(refNode,ctx):props.children&&(result=extractNode(props.children,ctx)),visitedRefs.delete(refId),result}let children=props.children,content2=children?extractNode(children,ctx):"";switch(tag){case"h1":return`# ${content2.trim()}
@@ -80,27 +79,29 @@ ${answer}
 Include the page title, all text content, code blocks, and tables.
 Do not summarize \u2014 extract the full content.
 
-URL: `;function shouldRethrow2(err){let message=err instanceof Error?err.message:String(err);return err instanceof CredentialResolutionError||message.startsWith("Failed to parse ")}async function extractWithUrlContext(url,signal){let requestSignal2=AbortSignal.any([AbortSignal.timeout(6e4),...signal?[signal]:[]]),apiKey=isGeminiAdcAvailable()?null:await getApiKey(requestSignal2);if(!apiKey&&!isGatewayConfigured()&&!isGeminiAdcAvailable())return null;let activityId=activityMonitor.logStart({type:"api",query:`url_context: ${url}`});try{let model=DEFAULT_MODEL,body={contents:[{role:"user",parts:[{text:EXTRACTION_PROMPT+url}]}],tools:[{url_context:{}}]},res=await fetchGeminiApi(`${getVersionedApiBase()}/models/${model}:generateContent`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(body),signal:requestSignal2},apiKey);if(!res.ok)return activityMonitor.logComplete(activityId,res.status),null;let data=await res.json();activityMonitor.logComplete(activityId,res.status);let metadata=data.candidates?.[0]?.url_context_metadata;if(metadata?.url_metadata?.length){let status=metadata.url_metadata[0].url_retrieval_status;if(status==="URL_RETRIEVAL_STATUS_UNSAFE"||status==="URL_RETRIEVAL_STATUS_ERROR")return null}let content=data.candidates?.[0]?.content?.parts?.map(p=>p.text).filter(Boolean).join(`
-`)??"";if(!content||content.length<50)return null;let title=extractTitleFromContent(content,url);return{url,title,content,error:null}}catch(err){if(shouldRethrow2(err))throw err;let message=err instanceof Error?err.message:String(err);return message.toLowerCase().includes("abort")?activityMonitor.logComplete(activityId,0):activityMonitor.logError(activityId,message),null}}async function extractWithGeminiWeb(url,signal){let cookies=await isGeminiWebAvailable();if(!cookies)return null;let activityId=activityMonitor.logStart({type:"api",query:`gemini_web: ${url}`});try{let text=await queryWithCookies(EXTRACTION_PROMPT+url,cookies,{signal,timeoutMs:6e4});if(activityMonitor.logComplete(activityId,200),!text||text.length<50)return null;let title=extractTitleFromContent(text,url);return{url,title,content:text,error:null}}catch(err){if(shouldRethrow2(err))throw err;let message=err instanceof Error?err.message:String(err);return message.toLowerCase().includes("abort")?activityMonitor.logComplete(activityId,0):activityMonitor.logError(activityId,message),null}}function extractTitleFromContent(text,url){return extractHeadingTitle(text)??(new URL(url).pathname.split("/").pop()||url)}import{existsSync as existsSync6,readFileSync as readFileSync6}from"node:fs";var CONFIG_PATH6=getWebSearchConfigPath(),BRIGHTDATA_REQUEST_URL="https://api.brightdata.com/request",EXTRACT_TIMEOUT_MS=6e4,DEFAULT_MAX_REDIRECTS=5,REDIRECT_STATUSES=new Set([301,302,303,307,308]),ZONE_PATTERN=/^[a-z0-9_-]+$/i,cachedConfig4=null;function parseFailureDetail(err){let position=(err instanceof Error?err.message:String(err)).match(/at position \d+(?: \(line \d+ column \d+\))?/);return position?`invalid JSON ${position[0]}`:"invalid JSON"}function loadConfig3(){if(cachedConfig4)return cachedConfig4;if(!existsSync6(CONFIG_PATH6))return cachedConfig4={},cachedConfig4;let raw=readFileSync6(CONFIG_PATH6,"utf8"),parsed;try{parsed=JSON.parse(raw)}catch(err){throw new Error(`Failed to parse ${CONFIG_PATH6}: ${parseFailureDetail(err)}`)}if(!parsed||typeof parsed!="object"||Array.isArray(parsed))throw new Error(`Invalid config in ${CONFIG_PATH6}: expected a JSON object`);return cachedConfig4=parsed,cachedConfig4}function normalizeZone(value){if(typeof value!="string")return null;let trimmed=value.trim();return trimmed&&ZONE_PATTERN.test(trimmed)?trimmed:null}function zoneSetting(){let fromEnv=process.env.BRIGHTDATA_UNLOCKER_ZONE;if(typeof fromEnv=="string"&&fromEnv.trim())return{raw:fromEnv.trim(),label:"BRIGHTDATA_UNLOCKER_ZONE"};let configured=loadConfig3().brightdataUnlockerZone;return typeof configured=="string"&&configured.trim()?{raw:configured.trim(),label:`brightdataUnlockerZone in ${CONFIG_PATH6}`}:null}function getZone(){let setting=zoneSetting();return setting?normalizeZone(setting.raw):null}function requireZone(){let setting=zoneSetting(),zone=setting?normalizeZone(setting.raw):null;if(zone)return zone;throw setting?new Error(`Invalid Bright Data Unlocker zone: ${setting.label} must be a zone name of letters, digits, "-", or "_". The zone must be of type "unblocker"; a SERP zone will not serve Web Unlocker requests.`):new Error(`Bright Data Web Unlocker zone not configured. Either:
-  1. Set brightdataUnlockerZone in ${CONFIG_PATH6}
+URL: `;function shouldRethrow2(err){let message=err instanceof Error?err.message:String(err);return err instanceof CredentialResolutionError||message.startsWith("Failed to parse ")}async function extractWithUrlContext(url,signal){let requestSignal3=AbortSignal.any([AbortSignal.timeout(6e4),...signal?[signal]:[]]),apiKey=isGeminiAdcAvailable()?null:await getApiKey(requestSignal3);if(!apiKey&&!isGatewayConfigured()&&!isGeminiAdcAvailable())return null;let activityId=activityMonitor.logStart({type:"api",query:`url_context: ${url}`});try{let model=DEFAULT_MODEL,body={contents:[{role:"user",parts:[{text:EXTRACTION_PROMPT+url}]}],tools:[{url_context:{}}]},res=await fetchGeminiApi(`${getVersionedApiBase()}/models/${model}:generateContent`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(body),signal:requestSignal3},apiKey);if(!res.ok)return activityMonitor.logComplete(activityId,res.status),null;let data=await res.json();activityMonitor.logComplete(activityId,res.status);let metadata=data.candidates?.[0]?.url_context_metadata;if(metadata?.url_metadata?.length){let status=metadata.url_metadata[0].url_retrieval_status;if(status==="URL_RETRIEVAL_STATUS_UNSAFE"||status==="URL_RETRIEVAL_STATUS_ERROR")return null}let content=data.candidates?.[0]?.content?.parts?.map(p=>p.text).filter(Boolean).join(`
+`)??"";if(!content||content.length<50)return null;let title=extractTitleFromContent(content,url);return{url,title,content,error:null}}catch(err){if(shouldRethrow2(err))throw err;let message=err instanceof Error?err.message:String(err);return message.toLowerCase().includes("abort")?activityMonitor.logComplete(activityId,0):activityMonitor.logError(activityId,message),null}}async function extractWithGeminiWeb(url,signal){let cookies=await isGeminiWebAvailable();if(!cookies)return null;let activityId=activityMonitor.logStart({type:"api",query:`gemini_web: ${url}`});try{let text=await queryWithCookies(EXTRACTION_PROMPT+url,cookies,{signal,timeoutMs:6e4});if(activityMonitor.logComplete(activityId,200),!text||text.length<50)return null;let title=extractTitleFromContent(text,url);return{url,title,content:text,error:null}}catch(err){if(shouldRethrow2(err))throw err;let message=err instanceof Error?err.message:String(err);return message.toLowerCase().includes("abort")?activityMonitor.logComplete(activityId,0):activityMonitor.logError(activityId,message),null}}function extractTitleFromContent(text,url){return extractHeadingTitle(text)??(new URL(url).pathname.split("/").pop()||url)}import{existsSync as existsSync6,readFileSync as readFileSync6}from"node:fs";import net from"node:net";var CONFIG_PATH6=getWebSearchConfigPath(),EXTRACT_TIMEOUT_MS=6e4,MARKDOWN_FILTER="fit",cachedConfig4=null;function loadConfig3(){if(cachedConfig4)return cachedConfig4;if(!existsSync6(CONFIG_PATH6))return cachedConfig4={},cachedConfig4;let raw=readFileSync6(CONFIG_PATH6,"utf8"),parsed;try{parsed=JSON.parse(raw)}catch(err){let message=err instanceof Error?err.message:String(err);throw new Error(`Failed to parse ${CONFIG_PATH6}: ${message}`)}if(!parsed||typeof parsed!="object"||Array.isArray(parsed))throw new Error(`Invalid config in ${CONFIG_PATH6}: expected a JSON object`);return cachedConfig4=parsed,cachedConfig4}function normalizeBaseUrl(value){if(typeof value!="string")return null;let trimmed=value.trim();if(!trimmed)return null;let parsed;try{parsed=new URL(trimmed)}catch{throw new Error(`Invalid Crawl4AI base URL in ${CONFIG_PATH6}: expected an HTTP or HTTPS URL`)}if(parsed.protocol!=="http:"&&parsed.protocol!=="https:")throw new Error(`Invalid Crawl4AI base URL in ${CONFIG_PATH6}: expected an HTTP or HTTPS URL`);if(parsed.username||parsed.password)throw new Error(`Invalid Crawl4AI base URL in ${CONFIG_PATH6}: URL credentials are not allowed`);return parsed.pathname=parsed.pathname.replace(/\/+$/,""),parsed.search="",parsed.hash="",parsed.toString().replace(/\/+$/,"")}function getBaseUrl(){return normalizeBaseUrl(process.env.CRAWL4AI_BASE_URL)??normalizeBaseUrl(loadConfig3().crawl4aiBaseUrl)}function requireBaseUrl(){let baseUrl=getBaseUrl();if(!baseUrl)throw new Error(`Crawl4AI base URL not configured. Either:
+  1. Set crawl4aiBaseUrl in ${CONFIG_PATH6}
+  2. Set CRAWL4AI_BASE_URL environment variable`);return baseUrl}async function getApiToken(signal){return resolveCredential({provider:"Crawl4AI",configuredValue:loadConfig3().crawl4aiApiToken,environmentValue:process.env.CRAWL4AI_API_TOKEN,signal})}function requestSignal(timeoutMs,signal){let timeout=AbortSignal.timeout(timeoutMs);return signal?AbortSignal.any([timeout,signal]):timeout}function errorMessage3(err){return err instanceof Error?err.message:String(err)}function isAbortException(err){return err instanceof DOMException&&(err.name==="AbortError"||err.name==="TimeoutError")}function ssrfOptions(options){return{...options?.ssrf??loadSsrfConfig(),...options?.lookup?{lookup:options.lookup}:{}}}function isLoopbackApiUrl(url){let hostname=url.hostname.toLowerCase().replace(/^\[|\]$/g,"").replace(/\.$/,"");return hostname==="localhost"||hostname==="::1"?!0:net.isIP(hostname)!==4?!1:hostname.split(".")[0]==="127"}function firstHeadingTitle(markdown){return/^[ \t]*#[ \t]+(\S.*?)[ \t\r]*$/m.exec(markdown)?.[1]??""}function isCrawl4aiAvailable(){return getBaseUrl()!==null}async function extractWithCrawl4ai(url,signal,options){let baseUrl=requireBaseUrl(),ssrf=ssrfOptions(options);await validateRemoteUrl(url,ssrf);let token=await getApiToken(signal),headers={"Content-Type":"application/json"};token&&(headers.Authorization=`Bearer ${token}`);let requestUrl=new URL(`${baseUrl}/md`),init={method:"POST",headers,body:JSON.stringify({url,f:MARKDOWN_FILTER}),signal:requestSignal(options?.timeoutMs??EXTRACT_TIMEOUT_MS,signal)},seeOther=!1,activityId=activityMonitor.logStart({type:"fetch",url:requestUrl.toString()});try{let response=await fetchRemoteUrl(requestUrl,init,{...ssrf,allowLoopback:isLoopbackApiUrl(requestUrl),onRedirect:({to,init:redirectInit,response:response2})=>{if(to.origin!==requestUrl.origin)throw new Error(`Crawl4AI refused cross-origin redirect to ${to.origin}`);return response2.status===303&&(seeOther=!0),!seeOther&&(response2.status===301||response2.status===302)?init:redirectInit}});if(!response.ok){let text=await response.text().catch(()=>"");throw new Error(`Crawl4AI md error ${response.status}: ${redactCredential(text,token).slice(0,300)}`)}let data;try{data=await response.json()}catch{throw new Error("Crawl4AI md returned invalid JSON")}if(!data||typeof data!="object"||Array.isArray(data))throw new Error("Crawl4AI md returned an unexpected response shape");let envelope=data;if(envelope.success===!1){let detail=typeof envelope.error=="string"?envelope.error:typeof envelope.detail=="string"?envelope.detail:"";throw new Error(`Crawl4AI md unsuccessful: ${redactCredential(detail.trim()||"unknown error",token)}`)}if(envelope.success!==!0)throw new Error("Crawl4AI md returned an unexpected response shape");if(typeof envelope.markdown!="string")throw new Error("Crawl4AI md returned markdown in an unexpected shape");activityMonitor.logComplete(activityId,response.status);let content=envelope.markdown.trim();return content?{url,title:firstHeadingTitle(content),content,error:null}:null}catch(err){throw signal?.aborted||isAbortException(err)?activityMonitor.logComplete(activityId,0):activityMonitor.logError(activityId,errorMessage3(err)),err}}import{existsSync as existsSync7,readFileSync as readFileSync7}from"node:fs";var CONFIG_PATH7=getWebSearchConfigPath(),BRIGHTDATA_REQUEST_URL="https://api.brightdata.com/request",EXTRACT_TIMEOUT_MS2=6e4,DEFAULT_MAX_REDIRECTS=5,REDIRECT_STATUSES=new Set([301,302,303,307,308]),ZONE_PATTERN=/^[a-z0-9_-]+$/i,cachedConfig5=null;function parseFailureDetail(err){let position=(err instanceof Error?err.message:String(err)).match(/at position \d+(?: \(line \d+ column \d+\))?/);return position?`invalid JSON ${position[0]}`:"invalid JSON"}function loadConfig4(){if(cachedConfig5)return cachedConfig5;if(!existsSync7(CONFIG_PATH7))return cachedConfig5={},cachedConfig5;let raw=readFileSync7(CONFIG_PATH7,"utf8"),parsed;try{parsed=JSON.parse(raw)}catch(err){throw new Error(`Failed to parse ${CONFIG_PATH7}: ${parseFailureDetail(err)}`)}if(!parsed||typeof parsed!="object"||Array.isArray(parsed))throw new Error(`Invalid config in ${CONFIG_PATH7}: expected a JSON object`);return cachedConfig5=parsed,cachedConfig5}function normalizeZone(value){if(typeof value!="string")return null;let trimmed=value.trim();return trimmed&&ZONE_PATTERN.test(trimmed)?trimmed:null}function zoneSetting(){let fromEnv=process.env.BRIGHTDATA_UNLOCKER_ZONE;if(typeof fromEnv=="string"&&fromEnv.trim())return{raw:fromEnv.trim(),label:"BRIGHTDATA_UNLOCKER_ZONE"};let configured=loadConfig4().brightdataUnlockerZone;return typeof configured=="string"&&configured.trim()?{raw:configured.trim(),label:`brightdataUnlockerZone in ${CONFIG_PATH7}`}:null}function getZone(){let setting=zoneSetting();return setting?normalizeZone(setting.raw):null}function requireZone(){let setting=zoneSetting(),zone=setting?normalizeZone(setting.raw):null;if(zone)return zone;throw setting?new Error(`Invalid Bright Data Unlocker zone: ${setting.label} must be a zone name of letters, digits, "-", or "_". The zone must be of type "unblocker"; a SERP zone will not serve Web Unlocker requests.`):new Error(`Bright Data Web Unlocker zone not configured. Either:
+  1. Set brightdataUnlockerZone in ${CONFIG_PATH7}
   2. Set BRIGHTDATA_UNLOCKER_ZONE environment variable
-The zone must be of type "unblocker"; a SERP zone will not serve Web Unlocker requests.`)}async function getApiKey2(signal){let apiKey=await resolveCredential({provider:"Bright Data",configuredValue:loadConfig3().brightdataApiKey,environmentValue:process.env.BRIGHTDATA_API_KEY,signal});if(!apiKey)throw new Error(`Bright Data API key not found. Either:
-  1. Create ${CONFIG_PATH6} with { "brightdataApiKey": "your-key" }
+The zone must be of type "unblocker"; a SERP zone will not serve Web Unlocker requests.`)}async function getApiKey2(signal){let apiKey=await resolveCredential({provider:"Bright Data",configuredValue:loadConfig4().brightdataApiKey,environmentValue:process.env.BRIGHTDATA_API_KEY,signal});if(!apiKey)throw new Error(`Bright Data API key not found. Either:
+  1. Create ${CONFIG_PATH7} with { "brightdataApiKey": "your-key" }
   2. Set BRIGHTDATA_API_KEY environment variable
-Get a key at https://brightdata.com/cp/setting/users`);return apiKey}function requestSignal(timeoutMs,signal){let timeout=AbortSignal.timeout(timeoutMs);return signal?AbortSignal.any([timeout,signal]):timeout}function errorMessage3(err){return err instanceof Error?err.message:String(err)}function isAbortError2(err){return errorMessage3(err).toLowerCase().includes("abort")}function ssrfOptions(options){return{allowRanges:options?.ssrf?.allowRanges??[],trustEnvProxy:options?.ssrf?.trustEnvProxy??!1,...options?.lookup?{lookup:options.lookup}:{}}}function withoutSensitiveHeaders(headers){let next={...headers};return delete next.Authorization,delete next.authorization,delete next.Cookie,delete next.cookie,delete next["X-API-Key"],delete next["x-api-key"],next}async function fetchBrightDataApi(url,init,options){let current=await validateRemoteUrl(url,ssrfOptions(options)),headers=init.headers;for(let redirects=0;;redirects++){let response=await fetch(current,{...init,headers,redirect:"manual"});if(!REDIRECT_STATUSES.has(response.status))return response;let location=response.headers.get("location");if(!location)return response;if(redirects===DEFAULT_MAX_REDIRECTS)throw new Error(`Too many redirects fetching ${current.toString()}`);let next=await validateRemoteUrl(new URL(location,current),ssrfOptions(options));next.origin!==current.origin&&(headers=withoutSensitiveHeaders(headers)),current=next}}function unlockerBody(url,zone){return{url,zone,format:"raw",data_format:"markdown"}}async function brightDataRequest(url,zone,signal,options){let apiKey=await getApiKey2(signal),headers={"Content-Type":"application/json",Authorization:`Bearer ${apiKey}`},activityId=activityMonitor.logStart({type:"fetch",url});try{let response=await fetchBrightDataApi(BRIGHTDATA_REQUEST_URL,{method:"POST",headers,body:JSON.stringify(unlockerBody(url,zone)),signal:requestSignal(options?.timeoutMs??EXTRACT_TIMEOUT_MS,signal)},options);if(!response.ok){let errorText=await response.text().catch(()=>"");throw new Error(`Bright Data Web Unlocker error ${response.status}: ${redactCredential(errorText,apiKey).slice(0,300)}`)}let text=await response.text();return activityMonitor.logComplete(activityId,response.status),text}catch(err){let message=errorMessage3(err),redactedMessage=redactCredential(message,apiKey);if(isAbortError2(err)?activityMonitor.logComplete(activityId,0):activityMonitor.logError(activityId,redactedMessage),redactedMessage===message)throw err;let redactedError=new Error(redactedMessage);throw err instanceof Error&&(redactedError.name=err.name),redactedError}}function headingTitle(text){let match=text.match(/^#{1,2}\s+(.+)/m);return match?match[1].replace(/\*+/g,"").trim():""}function isBrightDataUnlockerAvailable(){return getZone()===null?!1:hasCredentialSource({provider:"Bright Data",configuredValue:loadConfig3().brightdataApiKey,environmentValue:process.env.BRIGHTDATA_API_KEY})}async function extractWithBrightDataUnlocker(url,signal,options){let zone=requireZone();await validateRemoteUrl(url,ssrfOptions(options));let content=(await brightDataRequest(url,zone,signal,options)).trim();return content?{url,title:headingTitle(content),content,error:null}:null}import{execFileSync as execFileSync2}from"node:child_process";import{existsSync as existsSync7,readFileSync as readFileSync7,readdirSync,statSync}from"node:fs";import{readFile}from"node:fs/promises";import{resolve,extname,basename as basename2,join as join2,dirname}from"node:path";var CONFIG_PATH7=getWebSearchConfigPath(),DEFAULT_VIDEO_PROMPT=`Extract the complete content of this video. Include:
+Get a key at https://brightdata.com/cp/setting/users`);return apiKey}function requestSignal2(timeoutMs,signal){let timeout=AbortSignal.timeout(timeoutMs);return signal?AbortSignal.any([timeout,signal]):timeout}function errorMessage4(err){return err instanceof Error?err.message:String(err)}function isAbortError2(err){return errorMessage4(err).toLowerCase().includes("abort")}function ssrfOptions2(options){return{allowRanges:options?.ssrf?.allowRanges??[],trustEnvProxy:options?.ssrf?.trustEnvProxy??!1,...options?.lookup?{lookup:options.lookup}:{}}}function withoutSensitiveHeaders(headers){let next={...headers};return delete next.Authorization,delete next.authorization,delete next.Cookie,delete next.cookie,delete next["X-API-Key"],delete next["x-api-key"],next}async function fetchBrightDataApi(url,init,options){let current=await validateRemoteUrl(url,ssrfOptions2(options)),headers=init.headers;for(let redirects=0;;redirects++){let response=await fetch(current,{...init,headers,redirect:"manual"});if(!REDIRECT_STATUSES.has(response.status))return response;let location=response.headers.get("location");if(!location)return response;if(redirects===DEFAULT_MAX_REDIRECTS)throw new Error(`Too many redirects fetching ${current.toString()}`);let next=await validateRemoteUrl(new URL(location,current),ssrfOptions2(options));next.origin!==current.origin&&(headers=withoutSensitiveHeaders(headers)),current=next}}function unlockerBody(url,zone){return{url,zone,format:"raw",data_format:"markdown"}}async function brightDataRequest(url,zone,signal,options){let apiKey=await getApiKey2(signal),headers={"Content-Type":"application/json",Authorization:`Bearer ${apiKey}`},activityId=activityMonitor.logStart({type:"fetch",url});try{let response=await fetchBrightDataApi(BRIGHTDATA_REQUEST_URL,{method:"POST",headers,body:JSON.stringify(unlockerBody(url,zone)),signal:requestSignal2(options?.timeoutMs??EXTRACT_TIMEOUT_MS2,signal)},options);if(!response.ok){let errorText=await response.text().catch(()=>"");throw new Error(`Bright Data Web Unlocker error ${response.status}: ${redactCredential(errorText,apiKey).slice(0,300)}`)}let text=await response.text();return activityMonitor.logComplete(activityId,response.status),text}catch(err){let message=errorMessage4(err),redactedMessage=redactCredential(message,apiKey);if(isAbortError2(err)?activityMonitor.logComplete(activityId,0):activityMonitor.logError(activityId,redactedMessage),redactedMessage===message)throw err;let redactedError=new Error(redactedMessage);throw err instanceof Error&&(redactedError.name=err.name),redactedError}}function headingTitle(text){let match=text.match(/^#{1,2}\s+(.+)/m);return match?match[1].replace(/\*+/g,"").trim():""}function isBrightDataUnlockerAvailable(){return getZone()===null?!1:hasCredentialSource({provider:"Bright Data",configuredValue:loadConfig4().brightdataApiKey,environmentValue:process.env.BRIGHTDATA_API_KEY})}async function extractWithBrightDataUnlocker(url,signal,options){let zone=requireZone();await validateRemoteUrl(url,ssrfOptions2(options));let content=(await brightDataRequest(url,zone,signal,options)).trim();return content?{url,title:headingTitle(content),content,error:null}:null}import{execFileSync as execFileSync2}from"node:child_process";import{existsSync as existsSync8,readFileSync as readFileSync8,readdirSync,statSync}from"node:fs";import{readFile}from"node:fs/promises";import{resolve,extname,basename as basename2,join as join2,dirname}from"node:path";var CONFIG_PATH8=getWebSearchConfigPath(),DEFAULT_VIDEO_PROMPT=`Extract the complete content of this video. Include:
 1. Video title (infer from content if not explicit), duration
 2. A brief summary (2-3 sentences)
 3. Full transcript with timestamps
 4. Descriptions of any code, terminal commands, diagrams, slides, or UI shown on screen
 
-Format as markdown.`,VIDEO_EXTENSIONS={".mp4":"video/mp4",".mov":"video/quicktime",".webm":"video/webm",".avi":"video/x-msvideo",".mpeg":"video/mpeg",".mpg":"video/mpeg",".wmv":"video/x-ms-wmv",".flv":"video/x-flv",".3gp":"video/3gpp",".3gpp":"video/3gpp"};function shouldRethrow3(err){return(err instanceof Error?err.message:String(err)).startsWith("Failed to parse ")}function normalizePreferredModel2(value,fallback){if(typeof value!="string")return fallback;let normalized=value.trim();return normalized.length>0?normalized:fallback}function normalizeEnabled2(value,fallback){return typeof value=="boolean"?value:fallback}function normalizeMaxSizeMB(value,fallback){return typeof value!="number"||!Number.isFinite(value)?fallback:value>0?value:fallback}var VIDEO_CONFIG_DEFAULTS={enabled:!0,preferredModel:"gemini-3.6-flash",maxSizeMB:50},cachedVideoConfig=null;function loadVideoConfig(){if(cachedVideoConfig)return cachedVideoConfig;if(!existsSync7(CONFIG_PATH7))return cachedVideoConfig={...VIDEO_CONFIG_DEFAULTS},cachedVideoConfig;let rawText=readFileSync7(CONFIG_PATH7,"utf-8"),raw;try{raw=JSON.parse(rawText)}catch(err){let message=err instanceof Error?err.message:String(err);throw new Error(`Failed to parse ${CONFIG_PATH7}: ${message}`)}let v=raw.video??{};return cachedVideoConfig={enabled:normalizeEnabled2(v.enabled,VIDEO_CONFIG_DEFAULTS.enabled),preferredModel:normalizePreferredModel2(v.preferredModel,VIDEO_CONFIG_DEFAULTS.preferredModel),maxSizeMB:normalizeMaxSizeMB(v.maxSizeMB,VIDEO_CONFIG_DEFAULTS.maxSizeMB)},cachedVideoConfig}function isVideoFile(input){let config=loadVideoConfig();if(!config.enabled||!(input.startsWith("/")||input.startsWith("./")||input.startsWith("../")||input.startsWith("file://")))return null;let filePath=input;if(input.startsWith("file://"))try{filePath=decodeURIComponent(new URL(input).pathname)}catch{return null}let ext=extname(filePath).toLowerCase(),mimeType=VIDEO_EXTENSIONS[ext];if(!mimeType)return null;let absolutePath=resolveFilePath(filePath);if(!absolutePath)return null;let stat;try{stat=statSync(absolutePath)}catch{return null}if(!stat.isFile())return null;let maxBytes=config.maxSizeMB*1024*1024;return{absolutePath,mimeType,sizeBytes:stat.size,maxSizeBytes:maxBytes,withinUploadLimit:stat.size<=maxBytes}}function resolveFilePath(filePath){let absolutePath=resolve(filePath);if(existsSync7(absolutePath))return absolutePath;let dir=dirname(absolutePath),base=basename2(absolutePath);if(!existsSync7(dir))return null;try{let normalizedBase=normalizeSpaces(base),match=readdirSync(dir).find(f=>normalizeSpaces(f)===normalizedBase);return match?join2(dir,match):null}catch{return null}}function normalizeSpaces(s){return s.replace(/[\u00A0\u2000-\u200B\u202F\u205F\u3000\uFEFF]/g," ")}async function extractVideo(info,signal,options){let config=loadVideoConfig(),effectivePrompt=options?.prompt??DEFAULT_VIDEO_PROMPT,effectiveModel=options?.model??config.preferredModel,displayName=basename2(info.absolutePath);if(!info.withinUploadLimit){let sizeMB=(info.sizeBytes/1024/1024).toFixed(1),maxSizeMB=(info.maxSizeBytes/1024/1024).toFixed(1),error=`Local video ${displayName} is ${sizeMB} MiB, above configured video.maxSizeMB (${maxSizeMB} MiB) for Gemini analysis. Use timestamp/frames for ffmpeg frame extraction, increase video.maxSizeMB, or compress the file.`;return{url:info.absolutePath,title:displayName,content:error,error}}let activityId=activityMonitor.logStart({type:"fetch",url:`video:${displayName}`}),result=await tryVideoGeminiApi(info,effectivePrompt,effectiveModel,signal)??await tryVideoGeminiWeb(info,effectivePrompt,effectiveModel,signal);if(result){if(canAttachImages()){let thumbnail=await extractVideoFrame(info.absolutePath);"error"in thumbnail||(result.thumbnail=thumbnail)}return activityMonitor.logComplete(activityId,200),result}return signal?.aborted?(activityMonitor.logComplete(activityId,0),null):(activityMonitor.logError(activityId,"all video extraction paths failed"),null)}function mapFfprobeError(err){let{code,stderr,message}=readExecError(err);if(code==="ENOENT")return"ffprobe is not installed. Install ffmpeg which includes ffprobe";let snippet=trimErrorText(stderr||message);return snippet?`ffprobe failed: ${snippet}`:"ffprobe failed"}async function extractVideoFrame(filePath,seconds=1){try{let buffer=execFileSync2("ffmpeg",["-ss",String(seconds),"-i",filePath,"-frames:v","1","-f","image2pipe","-vcodec","mjpeg","pipe:1"],{maxBuffer:5242880,timeout:1e4,stdio:["pipe","pipe","pipe"]});return buffer.length===0?{error:"ffmpeg failed: empty output"}:{data:buffer.toString("base64"),mimeType:"image/jpeg"}}catch(err){return{error:mapFfmpegError(err)}}}async function getLocalVideoDuration(filePath){try{let output=execFileSync2("ffprobe",["-v","quiet","-show_entries","format=duration","-of","csv=p=0",filePath],{timeout:1e4,encoding:"utf-8",stdio:["pipe","pipe","pipe"]}).trim(),duration=Number.parseFloat(output);return Number.isFinite(duration)?duration:{error:"ffprobe failed: invalid duration output"}}catch(err){return{error:mapFfprobeError(err)}}}async function tryVideoGeminiWeb(info,prompt,model,signal){try{let cookies=await isGeminiWebAvailable();if(!cookies||signal?.aborted)return null;let text=await queryWithCookies(prompt,cookies,{files:[info.absolutePath],...model!=="gemini-3.6-flash"?{model}:{},signal,timeoutMs:18e4});return{url:info.absolutePath,title:extractVideoTitle(text,info.absolutePath),content:text,error:null}}catch(err){if(shouldRethrow3(err))throw err;return null}}async function tryVideoGeminiApi(info,prompt,model,signal){let apiKey=await getApiKey(signal);if(!apiKey||signal?.aborted)return null;let fileName=null;try{let uploaded=await uploadToFilesApi(info,apiKey,signal);fileName=uploaded.name,await pollFileState(fileName,apiKey,signal,12e4);let text=await queryGeminiApiWithVideo(prompt,uploaded.uri,{apiKey,model,mimeType:info.mimeType,signal,timeoutMs:12e4});return{url:info.absolutePath,title:extractVideoTitle(text,info.absolutePath),content:text,error:null}}catch(err){if(shouldRethrow3(err))throw err;return null}finally{fileName&&deleteGeminiFile(fileName,apiKey)}}async function uploadToFilesApi(info,apiKey,signal){let displayName=basename2(info.absolutePath),initRes=await fetchGeminiApi(`${getUploadBase()}/files`,{method:"POST",headers:{"X-Goog-Upload-Protocol":"resumable","X-Goog-Upload-Command":"start","X-Goog-Upload-Header-Content-Length":String(info.sizeBytes),"X-Goog-Upload-Header-Content-Type":info.mimeType,"Content-Type":"application/json"},body:JSON.stringify({file:{display_name:displayName}}),signal},apiKey);if(!initRes.ok){let text=redactGeminiApiResponse(initRes,await initRes.text(),apiKey);throw new Error(`File upload init failed: ${initRes.status} (${text.slice(0,200)})`)}let uploadUrl=initRes.headers.get("x-goog-upload-url");if(!uploadUrl)throw new Error("No upload URL in response headers");let fileData=await readFile(info.absolutePath),uploadRes=await fetchGeminiApi(uploadUrl,{method:"PUT",headers:{"Content-Length":String(info.sizeBytes),"X-Goog-Upload-Offset":"0","X-Goog-Upload-Command":"upload, finalize"},body:fileData,signal},apiKey);if(!uploadRes.ok){let text=redactGeminiApiResponse(uploadRes,await uploadRes.text(),apiKey);throw new Error(`File upload failed: ${uploadRes.status} (${text.slice(0,200)})`)}return(await uploadRes.json()).file}async function pollFileState(fileName,apiKey,signal,timeoutMs=12e4){let deadline=Date.now()+timeoutMs;for(;Date.now()<deadline;){if(signal?.aborted)throw new Error("Aborted");let res=await fetchGeminiApi(`${getVersionedApiBase()}/${fileName}`,{signal},apiKey);if(!res.ok)throw new Error(`File state check failed: ${res.status}`);let data=await res.json();if(data.state==="ACTIVE")return;if(data.state==="FAILED")throw new Error("File processing failed");await new Promise(r=>setTimeout(r,5e3))}throw new Error("File processing timed out")}function deleteGeminiFile(fileName,apiKey){fetchGeminiApi(`${getVersionedApiBase()}/${fileName}`,{method:"DELETE"},apiKey).catch(err=>{let message=err instanceof Error?err.message:String(err);console.error(`Failed to delete Gemini file ${fileName}: ${message}`)})}function extractVideoTitle(text,filePath){return extractHeadingTitle(text)??basename2(filePath,extname(filePath))}var DECLARATION_RELATIONS=new Set(["api-catalog","describedby","service-desc","service-doc","service-meta"]),RELATION_LABELS={"api-catalog":"API catalog",describedby:"Description","service-desc":"Service description","service-doc":"Service documentation","service-meta":"Service metadata"};function discoverDeclaredWebLinks(document,linkHeader,responseUrl){let links=new Map;for(let value of splitLinkHeader(linkHeader??"")){let target=/^\s*<([^>]*)>/.exec(value);if(!target)continue;let parameters=parseLinkParameters(value.slice(target[0].length));if(!(!parameters||parameters.has("anchor"))&&(addDeclaredLink(links,{url:resolveHttpUrl(target[1],responseUrl),relations:declaredRelations(parameters.get("rel")),type:parameters.get("type")}),links.size>=20))break}if(links.size<20){let declaredBase=document.querySelector("base[href]")?.getAttribute("href"),documentBase=resolveHttpUrl(declaredBase,responseUrl)??responseUrl;for(let element of document.querySelectorAll("link[rel][href], a[rel][href]"))if(addDeclaredLink(links,{url:resolveHttpUrl(element.getAttribute("href"),documentBase),relations:declaredRelations(element.getAttribute("rel")),type:element.getAttribute("type")}),links.size>=20)break}return[...links.values()]}function appendDeclaredWebLinks(content,links){if(links.length===0)return content;let section=["## Declared links","",...links.map(formatDeclaredLink)].join(`
+Format as markdown.`,VIDEO_EXTENSIONS={".mp4":"video/mp4",".mov":"video/quicktime",".webm":"video/webm",".avi":"video/x-msvideo",".mpeg":"video/mpeg",".mpg":"video/mpeg",".wmv":"video/x-ms-wmv",".flv":"video/x-flv",".3gp":"video/3gpp",".3gpp":"video/3gpp"};function shouldRethrow3(err){return(err instanceof Error?err.message:String(err)).startsWith("Failed to parse ")}function normalizePreferredModel2(value,fallback){if(typeof value!="string")return fallback;let normalized=value.trim();return normalized.length>0?normalized:fallback}function normalizeEnabled2(value,fallback){return typeof value=="boolean"?value:fallback}function normalizeMaxSizeMB(value,fallback){return typeof value!="number"||!Number.isFinite(value)?fallback:value>0?value:fallback}var VIDEO_CONFIG_DEFAULTS={enabled:!0,preferredModel:"gemini-3.6-flash",maxSizeMB:50},cachedVideoConfig=null;function loadVideoConfig(){if(cachedVideoConfig)return cachedVideoConfig;if(!existsSync8(CONFIG_PATH8))return cachedVideoConfig={...VIDEO_CONFIG_DEFAULTS},cachedVideoConfig;let rawText=readFileSync8(CONFIG_PATH8,"utf-8"),raw;try{raw=JSON.parse(rawText)}catch(err){let message=err instanceof Error?err.message:String(err);throw new Error(`Failed to parse ${CONFIG_PATH8}: ${message}`)}let v=raw.video??{};return cachedVideoConfig={enabled:normalizeEnabled2(v.enabled,VIDEO_CONFIG_DEFAULTS.enabled),preferredModel:normalizePreferredModel2(v.preferredModel,VIDEO_CONFIG_DEFAULTS.preferredModel),maxSizeMB:normalizeMaxSizeMB(v.maxSizeMB,VIDEO_CONFIG_DEFAULTS.maxSizeMB)},cachedVideoConfig}function isVideoFile(input){let config=loadVideoConfig();if(!config.enabled||!(input.startsWith("/")||input.startsWith("./")||input.startsWith("../")||input.startsWith("file://")))return null;let filePath=input;if(input.startsWith("file://"))try{filePath=decodeURIComponent(new URL(input).pathname)}catch{return null}let ext=extname(filePath).toLowerCase(),mimeType=VIDEO_EXTENSIONS[ext];if(!mimeType)return null;let absolutePath=resolveFilePath(filePath);if(!absolutePath)return null;let stat;try{stat=statSync(absolutePath)}catch{return null}if(!stat.isFile())return null;let maxBytes=config.maxSizeMB*1024*1024;return{absolutePath,mimeType,sizeBytes:stat.size,maxSizeBytes:maxBytes,withinUploadLimit:stat.size<=maxBytes}}function resolveFilePath(filePath){let absolutePath=resolve(filePath);if(existsSync8(absolutePath))return absolutePath;let dir=dirname(absolutePath),base=basename2(absolutePath);if(!existsSync8(dir))return null;try{let normalizedBase=normalizeSpaces(base),match=readdirSync(dir).find(f=>normalizeSpaces(f)===normalizedBase);return match?join2(dir,match):null}catch{return null}}function normalizeSpaces(s){return s.replace(/[\u00A0\u2000-\u200B\u202F\u205F\u3000\uFEFF]/g," ")}async function extractVideo(info,signal,options){let config=loadVideoConfig(),effectivePrompt=options?.prompt??DEFAULT_VIDEO_PROMPT,effectiveModel=options?.model??config.preferredModel,displayName=basename2(info.absolutePath);if(!info.withinUploadLimit){let sizeMB=(info.sizeBytes/1024/1024).toFixed(1),maxSizeMB=(info.maxSizeBytes/1024/1024).toFixed(1),error=`Local video ${displayName} is ${sizeMB} MiB, above configured video.maxSizeMB (${maxSizeMB} MiB) for Gemini analysis. Use timestamp/frames for ffmpeg frame extraction, increase video.maxSizeMB, or compress the file.`;return{url:info.absolutePath,title:displayName,content:error,error}}let activityId=activityMonitor.logStart({type:"fetch",url:`video:${displayName}`}),result=await tryVideoGeminiApi(info,effectivePrompt,effectiveModel,signal)??await tryVideoGeminiWeb(info,effectivePrompt,effectiveModel,signal);if(result){if(canAttachImages()){let thumbnail=await extractVideoFrame(info.absolutePath);"error"in thumbnail||(result.thumbnail=thumbnail)}return activityMonitor.logComplete(activityId,200),result}return signal?.aborted?(activityMonitor.logComplete(activityId,0),null):(activityMonitor.logError(activityId,"all video extraction paths failed"),null)}function mapFfprobeError(err){let{code,stderr,message}=readExecError(err);if(code==="ENOENT")return"ffprobe is not installed. Install ffmpeg which includes ffprobe";let snippet=trimErrorText(stderr||message);return snippet?`ffprobe failed: ${snippet}`:"ffprobe failed"}async function extractVideoFrame(filePath,seconds=1){try{let buffer=execFileSync2("ffmpeg",["-ss",String(seconds),"-i",filePath,"-frames:v","1","-f","image2pipe","-vcodec","mjpeg","pipe:1"],{maxBuffer:5242880,timeout:1e4,stdio:["pipe","pipe","pipe"]});return buffer.length===0?{error:"ffmpeg failed: empty output"}:{data:buffer.toString("base64"),mimeType:"image/jpeg"}}catch(err){return{error:mapFfmpegError(err)}}}async function getLocalVideoDuration(filePath){try{let output=execFileSync2("ffprobe",["-v","quiet","-show_entries","format=duration","-of","csv=p=0",filePath],{timeout:1e4,encoding:"utf-8",stdio:["pipe","pipe","pipe"]}).trim(),duration=Number.parseFloat(output);return Number.isFinite(duration)?duration:{error:"ffprobe failed: invalid duration output"}}catch(err){return{error:mapFfprobeError(err)}}}async function tryVideoGeminiWeb(info,prompt,model,signal){try{let cookies=await isGeminiWebAvailable();if(!cookies||signal?.aborted)return null;let text=await queryWithCookies(prompt,cookies,{files:[info.absolutePath],...model!=="gemini-3.6-flash"?{model}:{},signal,timeoutMs:18e4});return{url:info.absolutePath,title:extractVideoTitle(text,info.absolutePath),content:text,error:null}}catch(err){if(shouldRethrow3(err))throw err;return null}}async function tryVideoGeminiApi(info,prompt,model,signal){let apiKey=await getApiKey(signal);if(!apiKey||signal?.aborted)return null;let fileName=null;try{let uploaded=await uploadToFilesApi(info,apiKey,signal);fileName=uploaded.name,await pollFileState(fileName,apiKey,signal,12e4);let text=await queryGeminiApiWithVideo(prompt,uploaded.uri,{apiKey,model,mimeType:info.mimeType,signal,timeoutMs:12e4});return{url:info.absolutePath,title:extractVideoTitle(text,info.absolutePath),content:text,error:null}}catch(err){if(shouldRethrow3(err))throw err;return null}finally{fileName&&deleteGeminiFile(fileName,apiKey)}}async function uploadToFilesApi(info,apiKey,signal){let displayName=basename2(info.absolutePath),initRes=await fetchGeminiApi(`${getUploadBase()}/files`,{method:"POST",headers:{"X-Goog-Upload-Protocol":"resumable","X-Goog-Upload-Command":"start","X-Goog-Upload-Header-Content-Length":String(info.sizeBytes),"X-Goog-Upload-Header-Content-Type":info.mimeType,"Content-Type":"application/json"},body:JSON.stringify({file:{display_name:displayName}}),signal},apiKey);if(!initRes.ok){let text=redactGeminiApiResponse(initRes,await initRes.text(),apiKey);throw new Error(`File upload init failed: ${initRes.status} (${text.slice(0,200)})`)}let uploadUrl=initRes.headers.get("x-goog-upload-url");if(!uploadUrl)throw new Error("No upload URL in response headers");let fileData=await readFile(info.absolutePath),uploadRes=await fetchGeminiApi(uploadUrl,{method:"PUT",headers:{"Content-Length":String(info.sizeBytes),"X-Goog-Upload-Offset":"0","X-Goog-Upload-Command":"upload, finalize"},body:fileData,signal},apiKey);if(!uploadRes.ok){let text=redactGeminiApiResponse(uploadRes,await uploadRes.text(),apiKey);throw new Error(`File upload failed: ${uploadRes.status} (${text.slice(0,200)})`)}return(await uploadRes.json()).file}async function pollFileState(fileName,apiKey,signal,timeoutMs=12e4){let deadline=Date.now()+timeoutMs;for(;Date.now()<deadline;){if(signal?.aborted)throw new Error("Aborted");let res=await fetchGeminiApi(`${getVersionedApiBase()}/${fileName}`,{signal},apiKey);if(!res.ok)throw new Error(`File state check failed: ${res.status}`);let data=await res.json();if(data.state==="ACTIVE")return;if(data.state==="FAILED")throw new Error("File processing failed");await new Promise(r=>setTimeout(r,5e3))}throw new Error("File processing timed out")}function deleteGeminiFile(fileName,apiKey){fetchGeminiApi(`${getVersionedApiBase()}/${fileName}`,{method:"DELETE"},apiKey).catch(err=>{let message=err instanceof Error?err.message:String(err);console.error(`Failed to delete Gemini file ${fileName}: ${message}`)})}function extractVideoTitle(text,filePath){return extractHeadingTitle(text)??basename2(filePath,extname(filePath))}var DECLARATION_RELATIONS=new Set(["api-catalog","describedby","service-desc","service-doc","service-meta"]),RELATION_LABELS={"api-catalog":"API catalog",describedby:"Description","service-desc":"Service description","service-doc":"Service documentation","service-meta":"Service metadata"};function discoverDeclaredWebLinks(document,linkHeader,responseUrl){let links=new Map;for(let value of splitLinkHeader(linkHeader??"")){let target=/^\s*<([^>]*)>/.exec(value);if(!target)continue;let parameters=parseLinkParameters(value.slice(target[0].length));if(!(!parameters||parameters.has("anchor"))&&(addDeclaredLink(links,{url:resolveHttpUrl(target[1],responseUrl),relations:declaredRelations(parameters.get("rel")),type:parameters.get("type")}),links.size>=20))break}if(links.size<20){let declaredBase=document.querySelector("base[href]")?.getAttribute("href"),documentBase=resolveHttpUrl(declaredBase,responseUrl)??responseUrl;for(let element of document.querySelectorAll("link[rel][href], a[rel][href]"))if(addDeclaredLink(links,{url:resolveHttpUrl(element.getAttribute("href"),documentBase),relations:declaredRelations(element.getAttribute("rel")),type:element.getAttribute("type")}),links.size>=20)break}return[...links.values()]}function appendDeclaredWebLinks(content,links){if(links.length===0)return content;let section=["## Declared links","",...links.map(formatDeclaredLink)].join(`
 `);return content.trim()?`${content.trim()}
 
-${section}`:section}function addDeclaredLink(links,candidate){if(!candidate.url||candidate.relations.length===0)return;let existing=links.get(candidate.url);if(existing){for(let relation of candidate.relations)existing.relations.includes(relation)||existing.relations.push(relation);existing.type||(existing.type=normalizeMetadata(candidate.type));return}if(links.size>=20)return;let type=normalizeMetadata(candidate.type);links.set(candidate.url,{url:candidate.url,relations:candidate.relations,...type?{type}:{}})}function declaredRelations(value){return value?[...new Set(value.trim().toLowerCase().split(/\s+/).filter(relation=>DECLARATION_RELATIONS.has(relation)))]:[]}function resolveHttpUrl(value,baseUrl){if(!value||value.length>4096)return null;try{let url=new URL(value,baseUrl);return url.href.length>4096?null:url.protocol==="http:"||url.protocol==="https:"?url.href:null}catch{return null}}function splitOutsideSyntax(input,separator,protectTargets){let parts=[],start=0,inTarget=!1,inQuotes=!1,escaped=!1;for(let index=0;index<input.length;index++){let character=input[index];if(inQuotes){escaped?escaped=!1:character==="\\"?escaped=!0:character==='"'&&(inQuotes=!1);continue}character==='"'&&!inTarget?inQuotes=!0:protectTargets&&character==="<"?inTarget=!0:protectTargets&&character===">"?inTarget=!1:character===separator&&!inTarget&&(parts.push(input.slice(start,index)),start=index+1)}return inQuotes||inTarget?null:(parts.push(input.slice(start)),parts)}function splitLinkHeader(header){return(splitOutsideSyntax(header,",",!0)??[]).map(value=>value.trim()).filter(Boolean)}function parseLinkParameters(input){let parts=splitOutsideSyntax(input,";",!1);if(!parts||parts.shift()?.trim())return null;let parameters=new Map;for(let part of parts){let match=/^\s*([!#$%&'*+\-.^_`|~A-Za-z0-9]+)(?:\s*=\s*(?:"((?:\\.|[^"])*)"|(\S+)))?\s*$/.exec(part);if(!match)return null;let name=match[1].toLowerCase(),value=match[2]===void 0?match[3]??"":match[2].replace(/\\(.)/g,"$1");parameters.has(name)||parameters.set(name,value)}return parameters}function normalizeMetadata(value){return value&&value.replace(/\s+/g," ").trim().slice(0,160)||void 0}function formatDeclaredLink(link){let relation=link.relations.map(inlineCode).join(", "),type=link.type?`; ${inlineCode(link.type)}`:"";return`- ${RELATION_LABELS[link.relations[0]]??"Declared link"} (${relation}${type}): <${link.url}>`}function inlineCode(value){return`\`${value.replace(/`/g,"'")}\``}import{Buffer as Buffer3}from"node:buffer";import{createHash}from"node:crypto";var MAX_DATA_URI_HEADER_CHARS=1024,MAX_MIME_CHARS=127,MAX_MARKER_SOURCE_CHARS=180;function sha256Bytes(value){return createHash("sha256").update(value).digest("hex")}function asciiLowerCode(code){return code>=65&&code<=90?code+32:code}function matchesDataScheme(text,index){return index+5<=text.length&&asciiLowerCode(text.charCodeAt(index))===100&&asciiLowerCode(text.charCodeAt(index+1))===97&&asciiLowerCode(text.charCodeAt(index+2))===116&&asciiLowerCode(text.charCodeAt(index+3))===97&&text.charCodeAt(index+4)===58}function isSchemeBoundary(text,index){if(index===0)return!0;let code=text.charCodeAt(index-1);return!(code>=48&&code<=57||code>=65&&code<=90||code>=97&&code<=122)&&code!==43&&code!==45&&code!==46}function findNextDataScheme(text,from){for(let i=from;i+5<=text.length;i++)if(matchesDataScheme(text,i)&&isSchemeBoundary(text,i))return i;return-1}function dataUriEnclosure(text,start){if(start===0)return null;let immediateIndex=start-1,immediate=text[immediateIndex];if(immediate==='"'||immediate==="'"||immediate==="`"){let beforeQuote=immediateIndex>0?text[immediateIndex-1]:"";if(immediateIndex===0||beforeQuote==="="||beforeQuote==="("||beforeQuote==="["||beforeQuote==="{"||beforeQuote===":"||beforeQuote===","||/\s/.test(beforeQuote))return{kind:"quote",close:immediate}}let openerIndex=immediateIndex;for(;openerIndex>=0&&(text[openerIndex]===" "||text[openerIndex]==="	");)openerIndex--;let opener=text[openerIndex];return opener==="("?{kind:"parentheses"}:opener==="<"?{kind:"angle"}:null}function isBareDataUriTerminator(ch){let code=ch.charCodeAt(0);return code<=32||code===127||ch==='"'||ch==="`"||ch==="<"||ch===">"}function scanDataUriCandidate(text,start){let enclosure=dataUriEnclosure(text,start),comma=-1,depth=enclosure?.kind==="parentheses"?1:0,index=start+5;for(;index<text.length;index++){let ch=text[index];if(enclosure?.kind==="quote"){if(ch===enclosure.close)break}else if(enclosure?.kind==="parentheses"){if(ch==="(")depth++;else if(ch===")"&&(depth--,depth===0))break}else if(enclosure?.kind==="angle"){if(ch===">")break}else if(isBareDataUriTerminator(ch))break;comma<0&&ch===","&&(comma=index)}return{end:index,comma,enclosure}}function headerEndsWithBase64(text,headerStart,comma){let end=comma;for(;end>headerStart&&(text[end-1]===" "||text[end-1]==="	");)end--;let token="base64";if(end-headerStart<token.length+1)return!1;let tokenStart=end-token.length;for(let i=0;i<token.length;i++)if(asciiLowerCode(text.charCodeAt(tokenStart+i))!==token.charCodeAt(i))return!1;return tokenStart>headerStart&&text[tokenStart-1]===";"}function isMimeTokenChar(code){if(code<33||code>126)return!1;switch(code){case 34:case 40:case 41:case 44:case 47:case 58:case 59:case 60:case 61:case 62:case 63:case 64:case 91:case 92:case 93:return!1;default:return!0}}function normalizeMimeType(text,headerStart,comma){let end=headerStart;for(;end<comma&&text[end]!==";";)end++;if(end===headerStart)return"text/plain";if(end-headerStart>MAX_MIME_CHARS)return"application/octet-stream";let slashCount=0;for(let i=headerStart;i<end;i++){let code=text.charCodeAt(i);if(code===47){if(slashCount++,i===headerStart||i===end-1)return"application/octet-stream";continue}if(!isMimeTokenChar(code))return"application/octet-stream"}return slashCount!==1?"application/octet-stream":text.slice(headerStart,end).toLowerCase()}function hexValue(code){return code>=48&&code<=57?code-48:code>=65&&code<=70?code-55:code>=97&&code<=102?code-87:-1}function decodePercentEncoded(payload){let output=Buffer3.allocUnsafe(Buffer3.byteLength(payload,"utf8")),outputOffset=0,cursor=0;for(;cursor<payload.length;){let percent=payload.indexOf("%",cursor),runEnd=percent<0?payload.length:percent;if(runEnd>cursor){let run=Buffer3.from(payload.slice(cursor,runEnd),"utf8");run.copy(output,outputOffset),outputOffset+=run.length}if(percent<0)break;if(percent+2>=payload.length)return{error:"invalid-percent-escape"};let high=hexValue(payload.charCodeAt(percent+1)),low=hexValue(payload.charCodeAt(percent+2));if(high<0||low<0)return{error:"invalid-percent-escape"};output[outputOffset++]=high<<4|low,cursor=percent+3}return{bytes:output.subarray(0,outputOffset)}}function isBase64AlphabetByte(byte){return byte>=65&&byte<=90||byte>=97&&byte<=122||byte>=48&&byte<=57||byte===43||byte===47}function decodeBase64Payload(payload){let percentDecoded=decodePercentEncoded(payload);if("error"in percentDecoded)return percentDecoded;let encoded=percentDecoded.bytes,paddingStart=encoded.length,paddingCount=0;for(let i=0;i<encoded.length;i++){let byte=encoded[i];if(byte>127)return{error:"non-ascii-base64"};if(byte===61){paddingStart===encoded.length&&(paddingStart=i),paddingCount++;continue}if(paddingStart!==encoded.length)return{error:"invalid-base64-padding"};if(!isBase64AlphabetByte(byte))return{error:"invalid-base64-character"}}return paddingCount>2?{error:"invalid-base64-padding"}:paddingCount>0&&encoded.length%4!==0?{error:"invalid-base64-padding"}:paddingCount===0&&encoded.length%4===1?{error:"invalid-base64-length"}:paddingCount===1&&paddingStart%4!==3?{error:"invalid-base64-padding"}:paddingCount===2&&paddingStart%4!==2?{error:"invalid-base64-padding"}:{bytes:Buffer3.from(encoded.toString("ascii"),"base64")}}function safeMarkerSource(sourcePath){let source=sourcePath||"value",safe="";for(let ch of source){let code=ch.charCodeAt(0),allowed=code>=48&&code<=57||code>=65&&code<=90||code>=97&&code<=122||ch==="."||ch==="_"||ch==="-"||ch==="["||ch==="]"||ch==="*"||ch==="$";safe+=allowed?ch:"_"}if(safe.length<=MAX_MARKER_SOURCE_CHARS)return safe;let suffix=sha256Bytes(source).slice(0,12);return`${safe.slice(0,MAX_MARKER_SOURCE_CHARS-suffix.length-1)}~${suffix}`}function buildDataUriMarker(omission){let decoded=omission.decodedBytes===null?"unknown":String(omission.decodedBytes),decodeError=omission.decodeError?`; decodeError=${omission.decodeError}`:"";return`[pi-web-access inline data URI omitted; ordinal=${omission.ordinal}; source=${omission.sourcePath}; mime=${omission.mimeType}; encoding=${omission.encoding}; encodedBytes=${omission.encodedBytes}; decodedBytes=${decoded}${decodeError}; sha256=${omission.sha256}; digestBasis=${omission.digestBasis}; retrieval=not-retained]`}function sanitizeTextInternal(text,sourcePath,state){let scanFrom=0,retainedFrom=0,outputLength=0,pieces=[];for(;scanFrom<text.length;){let start=findNextDataScheme(text,scanFrom);if(start<0)break;let candidate=scanDataUriCandidate(text,start),end=candidate.end,headerStart=start+5,missingComma=candidate.comma<0;if(missingComma&&end===headerStart&&candidate.enclosure===null){scanFrom=end;continue}let headerEnd=missingComma?end:candidate.comma,headerLength=headerEnd-headerStart,encoding=headerEndsWithBase64(text,headerStart,headerEnd)?"base64":"percent-encoded",mimeType=normalizeMimeType(text,headerStart,headerEnd),payload=missingComma?text.slice(headerStart,end):text.slice(candidate.comma+1,end),encodedBytes=Buffer3.byteLength(payload,"utf8"),decodedBytes=null,decodeError,digestBasis="encoded",digest="";if(missingComma)decodeError="missing-comma",digest=sha256Bytes(payload);else if(headerLength>MAX_DATA_URI_HEADER_CHARS)decodeError="header-too-long",digest=sha256Bytes(payload);else{let decoded=encoding==="base64"?decodeBase64Payload(payload):decodePercentEncoded(payload);"error"in decoded?(decodeError=decoded.error,digest=sha256Bytes(payload)):(decodedBytes=decoded.bytes.length,digestBasis="decoded",digest=sha256Bytes(decoded.bytes))}let omission={ordinal:state.nextOrdinal++,sourcePath:safeMarkerSource(sourcePath),mimeType,encoding,encodedBytes,decodedBytes,...decodeError?{decodeError}:{},sha256:digest,digestBasis,retrieval:"not-retained"},marker=buildDataUriMarker(omission),prefix=text.slice(retainedFrom,start);pieces.push(prefix,marker),outputLength+=prefix.length;let markerStart=outputLength;outputLength+=marker.length,state.omissions.push({...omission,markerStart,markerEnd:outputLength}),retainedFrom=end,scanFrom=end}return state.omissions.length===0&&retainedFrom===0?text:(pieces.push(text.slice(retainedFrom)),pieces.join(""))}function sanitizeInlineDataUris(text,sourcePath){let state={nextOrdinal:1,omissions:[]};return{text:sanitizeTextInternal(text,sourcePath,state),omissions:state.omissions.map(({markerStart:_start,markerEnd:_end,...omission})=>omission)}}var DEFAULT_TIMEOUT_MS3=3e4,MAX_CONFIGURED_TIMEOUT_MS=2147483647,CONCURRENT_LIMIT=3,WEB_SEARCH_CONFIG_PATH=getWebSearchConfigPath();function loadFetchTimeoutMs(){if(!existsSync8(WEB_SEARCH_CONFIG_PATH))return DEFAULT_TIMEOUT_MS3;let raw;try{raw=JSON.parse(readFileSync8(WEB_SEARCH_CONFIG_PATH,"utf-8"))}catch(err){let message=err instanceof Error?err.message:String(err);throw new Error(`Failed to parse ${WEB_SEARCH_CONFIG_PATH}: ${message}`)}if(!raw||typeof raw!="object"||Array.isArray(raw))throw new Error(`Invalid config in ${WEB_SEARCH_CONFIG_PATH}: expected a JSON object`);let fetchConfig=raw.fetch;if(fetchConfig===void 0)return DEFAULT_TIMEOUT_MS3;if(!fetchConfig||typeof fetchConfig!="object"||Array.isArray(fetchConfig))throw new Error(`fetch in ${WEB_SEARCH_CONFIG_PATH} must be an object`);let value=fetchConfig.timeout;if(value===void 0)return DEFAULT_TIMEOUT_MS3;if(typeof value!="number"||!Number.isFinite(value)||value<=0)throw new Error(`Invalid fetch.timeout in ${WEB_SEARCH_CONFIG_PATH}: expected a positive finite number of seconds, got ${JSON.stringify(value)}`);let timeoutMs=Math.ceil(value*1e3);if(!Number.isSafeInteger(timeoutMs)||timeoutMs<1||timeoutMs>MAX_CONFIGURED_TIMEOUT_MS)throw new Error(`Invalid fetch.timeout in ${WEB_SEARCH_CONFIG_PATH}: converted timeout must be a finite safe integer from 1 through ${MAX_CONFIGURED_TIMEOUT_MS} milliseconds`);return Math.max(1,timeoutMs)}var NON_RECOVERABLE_ERRORS=["Unsupported content type","Response too large","PDF extraction is disabled","Image fetching is disabled"],MIN_USEFUL_CONTENT=500,SUPPORTED_IMAGE_TYPES=new Set(["image/png","image/jpeg","image/webp","image/gif"]),FETCH_PROVIDERS=["http","firecrawl","jina","tinyfish","search1api","querit","kagi","ollama","parallel","parallel-mcp","brightdata","gemini"],DEFAULT_FETCH_PROVIDER_ORDER=["http","firecrawl","jina","tinyfish","search1api","querit","kagi","ollama","parallel","brightdata","gemini"],REMOTE_HOSTED_FETCH_PROVIDERS=new Set(["jina","tinyfish","search1api","querit","kagi","ollama","parallel","parallel-mcp","brightdata","gemini"]);function isDefuddleConsoleError(args){let prefix=args[0];return prefix==="Defuddle"||typeof prefix=="string"&&/^Defuddle(?:\s|:)/.test(prefix)}async function extractWithDefuddle(text,url){let{Defuddle}=await import("./node-XY3CBY3C.js"),{document}=parseHTML(text);Object.defineProperty(document,"location",{value:new URL(url),configurable:!0});let processingError,originalConsoleError=console.error;console.error=(...args)=>{if(isDefuddleConsoleError(args)){args[0]==="Defuddle"&&args[1]==="Error processing document:"&&(processingError=args[2]);return}originalConsoleError(...args)};let resultPromise;try{resultPromise=Defuddle(document,url,{markdown:!0,useAsync:!1})}finally{console.error=originalConsoleError}let result=await resultPromise;if(processingError!==void 0)throw new Error(`Defuddle failed to process document: ${errorMessage4(processingError)}`);return typeof result.content=="string"?{title:result.title,content:result.content}:null}function loadSsrfAllowRanges(){return loadSsrfConfig().allowRanges}function errorMessage4(err){return err instanceof Error?err.message:String(err)}function isConfigParseError(err){return errorMessage4(err).startsWith("Failed to parse ")}function isAbortError3(err){return errorMessage4(err).toLowerCase().includes("abort")}function isRedirectPolicyError(message){return message.startsWith("Authenticated fetch refused cross-origin redirect")||message.startsWith("Blocked internal ")||message.startsWith("Blocked hostname by fetch_content domain policy")||message.startsWith("Hostname not allowed by fetch_content domain policy")||message.startsWith("Too many redirects fetching ")||message==="Only HTTP and HTTPS URLs can be fetched remotely"||message==="URL must include a hostname"||message.startsWith("Failed to resolve ")}function imageGateError(){try{return isImageEnabled()?null:"Image fetching is disabled by image.enabled"}catch(err){return errorMessage4(err)}}async function resolveAuthCookieHeader(url,profile){let parsed=assertAuthFetchUrl(profile,url.toString()),result=await getBrowserCookiesForHosts({hosts:[parsed.hostname],profile:profile.chromeProfile,requestUrl:parsed});if(result?.cookieHeader)return result.cookieHeader;if(!result){let diagnostic=getLastBrowserCookieDiagnostic();throw new Error(`Authenticated fetch profile ${profile.name} could not read browser cookies${diagnostic?`: ${diagnostic}`:""}`)}throw new Error(`Authenticated fetch profile ${profile.name} could not build a cookie header`)}var REDIRECT_STATUSES2=new Set([301,302,303,307,308]);async function fetchAuthenticatedRemoteUrl(url,init,validationOptions,profile){let current=await validateRemoteUrl(url,{allowRanges:validationOptions.ssrf.allowRanges,trustEnvProxy:validationOptions.ssrf.trustEnvProxy,domainPolicy:validationOptions.domainPolicy,...validationOptions.lookup?{lookup:validationOptions.lookup}:{}}),requestInit=init;for(let redirects=0;redirects<=5;redirects++){let cookieHeader=await resolveAuthCookieHeader(current,profile),headers={...requestInit.headers,cookie:cookieHeader},response=await fetch(current,{...requestInit,headers,redirect:"manual"});if(!REDIRECT_STATUSES2.has(response.status))return response;let location=response.headers.get("location");if(!location)return response;if(redirects===5)throw new Error(`Too many redirects fetching ${current.toString()}`);let from=current;if(current=await validateRemoteUrl(new URL(location,current),{allowRanges:validationOptions.ssrf.allowRanges,trustEnvProxy:validationOptions.ssrf.trustEnvProxy,domainPolicy:validationOptions.domainPolicy,...validationOptions.lookup?{lookup:validationOptions.lookup}:{}}),authFetchRedirectGuard(profile,from,current),response.status===303||(response.status===301||response.status===302)&&requestInit.method?.toUpperCase()==="POST"){let{body:_body,...nextInit}=requestInit;requestInit={...nextInit,method:"GET"}}}throw new Error(`Too many redirects fetching ${current.toString()}`)}function loadFetchRouting(){if(!existsSync8(WEB_SEARCH_CONFIG_PATH))return{providers:DEFAULT_FETCH_PROVIDER_ORDER,allowRemoteHostedProviders:!1};let raw;try{let parsed=JSON.parse(readFileSync8(WEB_SEARCH_CONFIG_PATH,"utf-8"));if(!parsed||typeof parsed!="object"||Array.isArray(parsed))throw new Error("expected a JSON object");raw=parsed}catch(err){let message=err instanceof Error?err.message:String(err);throw new Error(`Failed to parse ${WEB_SEARCH_CONFIG_PATH}: ${message}`)}if(!Object.hasOwn(raw,"fetchRouting"))return{providers:DEFAULT_FETCH_PROVIDER_ORDER,allowRemoteHostedProviders:!1};let routing=raw.fetchRouting;if(!routing||typeof routing!="object"||Array.isArray(routing))throw new Error(`fetchRouting in ${WEB_SEARCH_CONFIG_PATH} must be an object`);let routingConfig=routing,providersValue=routingConfig.providers,providers=DEFAULT_FETCH_PROVIDER_ORDER;if(providersValue!==void 0){if(!Array.isArray(providersValue)||providersValue.length===0)throw new Error(`fetchRouting.providers in ${WEB_SEARCH_CONFIG_PATH} must be a non-empty array`);providers=[];for(let provider of providersValue){let normalized=typeof provider=="string"?provider.trim().toLowerCase():"";if(!FETCH_PROVIDERS.includes(normalized))throw new Error(`fetchRouting.providers in ${WEB_SEARCH_CONFIG_PATH} contains an invalid provider: ${String(provider)}`);if(providers.includes(normalized))throw new Error(`fetchRouting.providers in ${WEB_SEARCH_CONFIG_PATH} must not contain duplicates: ${normalized}`);providers.push(normalized)}}let allowRemoteHostedProvidersValue=routingConfig.allowRemoteHostedProviders;if(allowRemoteHostedProvidersValue!==void 0&&typeof allowRemoteHostedProvidersValue!="boolean")throw new Error(`fetchRouting.allowRemoteHostedProviders in ${WEB_SEARCH_CONFIG_PATH} must be a boolean`);return{providers,allowRemoteHostedProviders:allowRemoteHostedProvidersValue===!0}}function notFoundGuidance(result,toolNames){let lines=[result.error??`HTTP ${result.status}`,"",`The origin server says this page does not exist (HTTP ${result.status}), so extraction providers cannot retrieve it.`];return toolNames?.webSearch&&toolNames.fetchContent?lines.push(`The page may have moved or been renamed. Use ${toolNames.webSearch} to find the current URL, then retry ${toolNames.fetchContent} with it.`):toolNames?.webSearch?lines.push(`The page may have moved or been renamed. Use ${toolNames.webSearch} to find the current URL.`):lines.push("The page may have moved or been renamed. Find the current URL, then retry the fetch with it."),lines.join(`
-`)}function abortedResult(url){return{url,title:"",content:"",error:"Aborted"}}var turndown=new import_turndown.default({headingStyle:"atx",codeBlockStyle:"fenced"}),fetchLimit=pLimit(CONCURRENT_LIMIT);function resolveFetchTimeoutMs(options){return options?.timeoutMs??loadFetchTimeoutMs()}var JINA_READER_BASE="https://r.jina.ai/";async function extractWithJinaReader(url,timeoutMs,signal,lookup){let jinaUrl=JINA_READER_BASE+url,activityId=activityMonitor.logStart({type:"api",query:`jina: ${url}`});try{let ssrf=loadSsrfConfig(),domainPolicy=loadFetchContentDomainPolicy();await validateRemoteUrl(url,{allowRanges:ssrf.allowRanges,trustEnvProxy:ssrf.trustEnvProxy,domainPolicy,...lookup?{lookup}:{}});let res=await fetch(jinaUrl,{headers:{Accept:"text/markdown","X-No-Cache":"true"},signal:AbortSignal.any([AbortSignal.timeout(timeoutMs),...signal?[signal]:[]])});if(!res.ok)return activityMonitor.logComplete(activityId,res.status),null;let content=await res.text();activityMonitor.logComplete(activityId,res.status);let contentStart=content.indexOf("Markdown Content:");if(contentStart<0)return null;let markdownPart=content.slice(contentStart+17).trim();if(markdownPart.length<100||markdownPart.startsWith("Loading...")||markdownPart.startsWith("Please enable JavaScript"))return null;let title=extractHeadingTitle(markdownPart)??(new URL(url).pathname.split("/").pop()||url);return{url,title,content:markdownPart,error:null}}catch(err){let message=err instanceof Error?err.message:String(err);return message.toLowerCase().includes("abort")?activityMonitor.logComplete(activityId,0):activityMonitor.logError(activityId,message),null}}function parseTimestamp(ts){let num=Number(ts);if(!isNaN(num)&&num>=0)return Math.floor(num);let parts=ts.split(":").map(Number);return parts.some(p=>isNaN(p)||p<0)?null:parts.length===3?Math.floor(parts[0]*3600+parts[1]*60+parts[2]):parts.length===2?Math.floor(parts[0]*60+parts[1]):null}function parseTimestampSpec(ts){let dashIdx=ts.indexOf("-",1);if(dashIdx>0){let start=parseTimestamp(ts.slice(0,dashIdx)),end=parseTimestamp(ts.slice(dashIdx+1));if(start!==null&&end!==null&&end>start)return{type:"range",start,end}}let seconds=parseTimestamp(ts);return seconds!==null?{type:"single",seconds}:null}var DEFAULT_RANGE_FRAMES=6,MIN_FRAME_INTERVAL=5;function computeRangeTimestamps(start,end,maxFrames=DEFAULT_RANGE_FRAMES){if(maxFrames<=1)return[start];let idealInterval=(end-start)/(maxFrames-1);if(idealInterval<MIN_FRAME_INTERVAL){let timestamps=[];for(let t=start;t<=end&&timestamps.length<maxFrames;t+=MIN_FRAME_INTERVAL)timestamps.push(t);return timestamps}return Array.from({length:maxFrames},(_,i)=>Math.round(start+i*idealInterval))}function buildFrameResult(url,label,requestedCount,frames,error,duration){if(frames.length===0){let msg=error??"Frame extraction failed";return{url,title:`Frames ${label} (0/${requestedCount})`,content:msg,error:msg}}return{url,title:`Frames ${label} (${frames.length}/${requestedCount})`,content:`${frames.length} frames extracted from ${label}`,error:null,frames,...duration!==void 0?{duration}:{}}}async function extractLocalFrames(filePath,timestamps){let results=await Promise.all(timestamps.map(async t=>{let frame=await extractVideoFrame(filePath,t);return"error"in frame?{error:frame.error}:{...frame,timestamp:formatSeconds(t)}})),frames=results.filter(f=>"data"in f),firstError=results.find(f=>"error"in f);return{frames,error:frames.length===0&&firstError?firstError.error:null}}function safeVideoInfo(url){try{let info=isVideoFile(url);return info?{status:"video",info}:{status:"not-video"}}catch(err){return{status:"invalid",error:errorMessage4(err)}}}async function extractContent(url,signal,options){if(signal?.aborted)return{url,title:"",content:"",error:"Aborted"};let remoteUrl=null;try{let parsed=new URL(url);(parsed.protocol==="http:"||parsed.protocol==="https:")&&(remoteUrl=parsed)}catch{}if(remoteUrl)try{let ssrf=loadSsrfConfig(),domainPolicy=loadFetchContentDomainPolicy();await validateRemoteUrl(remoteUrl,{allowRanges:ssrf.allowRanges,trustEnvProxy:ssrf.trustEnvProxy,domainPolicy,...options?.lookup?{lookup:options.lookup}:{}})}catch(err){return{url,title:"",content:"",error:errorMessage4(err)}}if(options?.authFetchProfile||options?.mode==="raw")try{return await extractViaHttp(url,resolveFetchTimeoutMs(options),signal,options)}catch(err){return{url,title:"",content:"",error:errorMessage4(err)}}if(options?.frames||options?.timestamp){let disabled=imageGateError();if(disabled)return{url,title:"",content:"",error:disabled}}if(options?.frames&&!options.timestamp){let frameCount=options.frames,ytInfo2=isYouTubeURL(url);if(ytInfo2.isYouTube&&ytInfo2.videoId){let streamInfo=await getYouTubeStreamInfo(ytInfo2.videoId);if("error"in streamInfo)return{url,title:"Frames",content:streamInfo.error,error:streamInfo.error};if(streamInfo.duration===null){let error="Cannot determine video duration. Use a timestamp range instead.";return{url,title:"Frames",content:error,error}}let dur=Math.floor(streamInfo.duration),timestamps=computeRangeTimestamps(0,dur,frameCount),result=await extractYouTubeFrames(ytInfo2.videoId,timestamps,streamInfo),label=`${formatSeconds(0)}-${formatSeconds(dur)}`;return buildFrameResult(url,label,timestamps.length,result.frames,result.error,streamInfo.duration)}let localVideo2=safeVideoInfo(url);if(localVideo2.status==="invalid")return{url,title:"",content:"",error:localVideo2.error};if(localVideo2.status==="video"){let durationResult=await getLocalVideoDuration(localVideo2.info.absolutePath);if(typeof durationResult!="number")return{url,title:"Frames",content:durationResult.error,error:durationResult.error};let dur=Math.floor(durationResult),timestamps=computeRangeTimestamps(0,dur,frameCount),result=await extractLocalFrames(localVideo2.info.absolutePath,timestamps),label=`${formatSeconds(0)}-${formatSeconds(dur)}`;return buildFrameResult(url,label,timestamps.length,result.frames,result.error,durationResult)}return{url,title:"",content:"",error:"Frame extraction only works with YouTube and local video files"}}if(options?.timestamp){let spec=parseTimestampSpec(options.timestamp);if(!spec)return{url,title:"",content:"",error:`Invalid timestamp format: "${options.timestamp}". Use "H:MM:SS", "MM:SS", "85", or "start-end".`};let frameCount=options.frames,ytInfo2=isYouTubeURL(url);if(ytInfo2.isYouTube&&ytInfo2.videoId){let streamInfo=await getYouTubeStreamInfo(ytInfo2.videoId);if("error"in streamInfo){if(spec.type==="range"){let label=`${formatSeconds(spec.start)}-${formatSeconds(spec.end)}`;return{url,title:`Frames ${label}`,content:streamInfo.error,error:streamInfo.error}}if(frameCount){let end=spec.seconds+(frameCount-1)*MIN_FRAME_INTERVAL,label=`${formatSeconds(spec.seconds)}-${formatSeconds(end)}`;return{url,title:`Frames ${label}`,content:streamInfo.error,error:streamInfo.error}}return{url,title:`Frame at ${options.timestamp}`,content:streamInfo.error,error:streamInfo.error}}if(spec.type==="range"){let label=`${formatSeconds(spec.start)}-${formatSeconds(spec.end)}`;if(streamInfo.duration!==null&&spec.end>streamInfo.duration){let error=`Timestamp ${formatSeconds(spec.end)} exceeds video duration (${formatSeconds(Math.floor(streamInfo.duration))})`;return{url,title:`Frames ${label}`,content:error,error}}let timestamps=frameCount?computeRangeTimestamps(spec.start,spec.end,frameCount):computeRangeTimestamps(spec.start,spec.end),result=await extractYouTubeFrames(ytInfo2.videoId,timestamps,streamInfo);return buildFrameResult(url,label,timestamps.length,result.frames,result.error,result.duration??void 0)}if(frameCount){let end=spec.seconds+(frameCount-1)*MIN_FRAME_INTERVAL,label=`${formatSeconds(spec.seconds)}-${formatSeconds(end)}`;if(streamInfo.duration!==null&&end>streamInfo.duration){let error=`Timestamp ${formatSeconds(end)} exceeds video duration (${formatSeconds(Math.floor(streamInfo.duration))})`;return{url,title:`Frames ${label}`,content:error,error}}let timestamps=computeRangeTimestamps(spec.seconds,end,frameCount),result=await extractYouTubeFrames(ytInfo2.videoId,timestamps,streamInfo);return buildFrameResult(url,label,timestamps.length,result.frames,result.error,result.duration??void 0)}if(streamInfo.duration!==null&&spec.seconds>streamInfo.duration){let error=`Timestamp ${formatSeconds(spec.seconds)} exceeds video duration (${formatSeconds(Math.floor(streamInfo.duration))})`;return{url,title:`Frame at ${options.timestamp}`,content:error,error}}let frame=await extractYouTubeFrame(ytInfo2.videoId,spec.seconds,streamInfo);return"error"in frame?{url,title:`Frame at ${options.timestamp}`,content:frame.error,error:frame.error}:{url,title:`Frame at ${options.timestamp}`,content:`Video frame at ${options.timestamp}`,error:null,thumbnail:frame}}let localVideo2=safeVideoInfo(url);if(localVideo2.status==="invalid")return{url,title:"",content:"",error:localVideo2.error};if(localVideo2.status==="video"){if(spec.type==="range"){let timestamps=frameCount?computeRangeTimestamps(spec.start,spec.end,frameCount):computeRangeTimestamps(spec.start,spec.end),result=await extractLocalFrames(localVideo2.info.absolutePath,timestamps),label=`${formatSeconds(spec.start)}-${formatSeconds(spec.end)}`;return buildFrameResult(url,label,timestamps.length,result.frames,result.error)}if(frameCount){let end=spec.seconds+(frameCount-1)*MIN_FRAME_INTERVAL,timestamps=computeRangeTimestamps(spec.seconds,end,frameCount),result=await extractLocalFrames(localVideo2.info.absolutePath,timestamps),label=`${formatSeconds(spec.seconds)}-${formatSeconds(end)}`;return buildFrameResult(url,label,timestamps.length,result.frames,result.error)}let frame=await extractVideoFrame(localVideo2.info.absolutePath,spec.seconds);return"error"in frame?{url,title:`Frame at ${options.timestamp}`,content:frame.error,error:frame.error}:{url,title:`Frame at ${options.timestamp}`,content:`Video frame at ${options.timestamp}`,error:null,thumbnail:frame}}return{url,title:"",content:"",error:"Timestamp extraction only works with YouTube and local video files"}}let localVideo=safeVideoInfo(url);if(localVideo.status==="invalid")return{url,title:"",content:"",error:localVideo.error};if(localVideo.status==="video")try{let result=await extractVideo(localVideo.info,signal,options);return signal?.aborted?abortedResult(url):result??{url,title:"",content:"",error:`Video analysis requires Gemini access. Either:
+${section}`:section}function addDeclaredLink(links,candidate){if(!candidate.url||candidate.relations.length===0)return;let existing=links.get(candidate.url);if(existing){for(let relation of candidate.relations)existing.relations.includes(relation)||existing.relations.push(relation);existing.type||(existing.type=normalizeMetadata(candidate.type));return}if(links.size>=20)return;let type=normalizeMetadata(candidate.type);links.set(candidate.url,{url:candidate.url,relations:candidate.relations,...type?{type}:{}})}function declaredRelations(value){return value?[...new Set(value.trim().toLowerCase().split(/\s+/).filter(relation=>DECLARATION_RELATIONS.has(relation)))]:[]}function resolveHttpUrl(value,baseUrl){if(!value||value.length>4096)return null;try{let url=new URL(value,baseUrl);return url.href.length>4096?null:url.protocol==="http:"||url.protocol==="https:"?url.href:null}catch{return null}}function splitOutsideSyntax(input,separator,protectTargets){let parts=[],start=0,inTarget=!1,inQuotes=!1,escaped=!1;for(let index=0;index<input.length;index++){let character=input[index];if(inQuotes){escaped?escaped=!1:character==="\\"?escaped=!0:character==='"'&&(inQuotes=!1);continue}character==='"'&&!inTarget?inQuotes=!0:protectTargets&&character==="<"?inTarget=!0:protectTargets&&character===">"?inTarget=!1:character===separator&&!inTarget&&(parts.push(input.slice(start,index)),start=index+1)}return inQuotes||inTarget?null:(parts.push(input.slice(start)),parts)}function splitLinkHeader(header){return(splitOutsideSyntax(header,",",!0)??[]).map(value=>value.trim()).filter(Boolean)}function parseLinkParameters(input){let parts=splitOutsideSyntax(input,";",!1);if(!parts||parts.shift()?.trim())return null;let parameters=new Map;for(let part of parts){let match=/^\s*([!#$%&'*+\-.^_`|~A-Za-z0-9]+)(?:\s*=\s*(?:"((?:\\.|[^"])*)"|(\S+)))?\s*$/.exec(part);if(!match)return null;let name=match[1].toLowerCase(),value=match[2]===void 0?match[3]??"":match[2].replace(/\\(.)/g,"$1");parameters.has(name)||parameters.set(name,value)}return parameters}function normalizeMetadata(value){return value&&value.replace(/\s+/g," ").trim().slice(0,160)||void 0}function formatDeclaredLink(link){let relation=link.relations.map(inlineCode).join(", "),type=link.type?`; ${inlineCode(link.type)}`:"";return`- ${RELATION_LABELS[link.relations[0]]??"Declared link"} (${relation}${type}): <${link.url}>`}function inlineCode(value){return`\`${value.replace(/`/g,"'")}\``}import{Buffer as Buffer3}from"node:buffer";import{createHash}from"node:crypto";var MAX_DATA_URI_HEADER_CHARS=1024,MAX_MIME_CHARS=127,MAX_MARKER_SOURCE_CHARS=180;function sha256Bytes(value){return createHash("sha256").update(value).digest("hex")}function asciiLowerCode(code){return code>=65&&code<=90?code+32:code}function matchesDataScheme(text,index){return index+5<=text.length&&asciiLowerCode(text.charCodeAt(index))===100&&asciiLowerCode(text.charCodeAt(index+1))===97&&asciiLowerCode(text.charCodeAt(index+2))===116&&asciiLowerCode(text.charCodeAt(index+3))===97&&text.charCodeAt(index+4)===58}function isSchemeBoundary(text,index){if(index===0)return!0;let code=text.charCodeAt(index-1);return!(code>=48&&code<=57||code>=65&&code<=90||code>=97&&code<=122)&&code!==43&&code!==45&&code!==46}function findNextDataScheme(text,from){for(let i=from;i+5<=text.length;i++)if(matchesDataScheme(text,i)&&isSchemeBoundary(text,i))return i;return-1}function dataUriEnclosure(text,start){if(start===0)return null;let immediateIndex=start-1,immediate=text[immediateIndex];if(immediate==='"'||immediate==="'"||immediate==="`"){let beforeQuote=immediateIndex>0?text[immediateIndex-1]:"";if(immediateIndex===0||beforeQuote==="="||beforeQuote==="("||beforeQuote==="["||beforeQuote==="{"||beforeQuote===":"||beforeQuote===","||/\s/.test(beforeQuote))return{kind:"quote",close:immediate}}let openerIndex=immediateIndex;for(;openerIndex>=0&&(text[openerIndex]===" "||text[openerIndex]==="	");)openerIndex--;let opener=text[openerIndex];return opener==="("?{kind:"parentheses"}:opener==="<"?{kind:"angle"}:null}function isBareDataUriTerminator(ch){let code=ch.charCodeAt(0);return code<=32||code===127||ch==='"'||ch==="`"||ch==="<"||ch===">"}function scanDataUriCandidate(text,start){let enclosure=dataUriEnclosure(text,start),comma=-1,depth=enclosure?.kind==="parentheses"?1:0,index=start+5;for(;index<text.length;index++){let ch=text[index];if(enclosure?.kind==="quote"){if(ch===enclosure.close)break}else if(enclosure?.kind==="parentheses"){if(ch==="(")depth++;else if(ch===")"&&(depth--,depth===0))break}else if(enclosure?.kind==="angle"){if(ch===">")break}else if(isBareDataUriTerminator(ch))break;comma<0&&ch===","&&(comma=index)}return{end:index,comma,enclosure}}function headerEndsWithBase64(text,headerStart,comma){let end=comma;for(;end>headerStart&&(text[end-1]===" "||text[end-1]==="	");)end--;let token="base64";if(end-headerStart<token.length+1)return!1;let tokenStart=end-token.length;for(let i=0;i<token.length;i++)if(asciiLowerCode(text.charCodeAt(tokenStart+i))!==token.charCodeAt(i))return!1;return tokenStart>headerStart&&text[tokenStart-1]===";"}function isMimeTokenChar(code){if(code<33||code>126)return!1;switch(code){case 34:case 40:case 41:case 44:case 47:case 58:case 59:case 60:case 61:case 62:case 63:case 64:case 91:case 92:case 93:return!1;default:return!0}}function normalizeMimeType(text,headerStart,comma){let end=headerStart;for(;end<comma&&text[end]!==";";)end++;if(end===headerStart)return"text/plain";if(end-headerStart>MAX_MIME_CHARS)return"application/octet-stream";let slashCount=0;for(let i=headerStart;i<end;i++){let code=text.charCodeAt(i);if(code===47){if(slashCount++,i===headerStart||i===end-1)return"application/octet-stream";continue}if(!isMimeTokenChar(code))return"application/octet-stream"}return slashCount!==1?"application/octet-stream":text.slice(headerStart,end).toLowerCase()}function hexValue(code){return code>=48&&code<=57?code-48:code>=65&&code<=70?code-55:code>=97&&code<=102?code-87:-1}function decodePercentEncoded(payload){let output=Buffer3.allocUnsafe(Buffer3.byteLength(payload,"utf8")),outputOffset=0,cursor=0;for(;cursor<payload.length;){let percent=payload.indexOf("%",cursor),runEnd=percent<0?payload.length:percent;if(runEnd>cursor){let run=Buffer3.from(payload.slice(cursor,runEnd),"utf8");run.copy(output,outputOffset),outputOffset+=run.length}if(percent<0)break;if(percent+2>=payload.length)return{error:"invalid-percent-escape"};let high=hexValue(payload.charCodeAt(percent+1)),low=hexValue(payload.charCodeAt(percent+2));if(high<0||low<0)return{error:"invalid-percent-escape"};output[outputOffset++]=high<<4|low,cursor=percent+3}return{bytes:output.subarray(0,outputOffset)}}function isBase64AlphabetByte(byte){return byte>=65&&byte<=90||byte>=97&&byte<=122||byte>=48&&byte<=57||byte===43||byte===47}function decodeBase64Payload(payload){let percentDecoded=decodePercentEncoded(payload);if("error"in percentDecoded)return percentDecoded;let encoded=percentDecoded.bytes,paddingStart=encoded.length,paddingCount=0;for(let i=0;i<encoded.length;i++){let byte=encoded[i];if(byte>127)return{error:"non-ascii-base64"};if(byte===61){paddingStart===encoded.length&&(paddingStart=i),paddingCount++;continue}if(paddingStart!==encoded.length)return{error:"invalid-base64-padding"};if(!isBase64AlphabetByte(byte))return{error:"invalid-base64-character"}}return paddingCount>2?{error:"invalid-base64-padding"}:paddingCount>0&&encoded.length%4!==0?{error:"invalid-base64-padding"}:paddingCount===0&&encoded.length%4===1?{error:"invalid-base64-length"}:paddingCount===1&&paddingStart%4!==3?{error:"invalid-base64-padding"}:paddingCount===2&&paddingStart%4!==2?{error:"invalid-base64-padding"}:{bytes:Buffer3.from(encoded.toString("ascii"),"base64")}}function safeMarkerSource(sourcePath){let source=sourcePath||"value",safe="";for(let ch of source){let code=ch.charCodeAt(0),allowed=code>=48&&code<=57||code>=65&&code<=90||code>=97&&code<=122||ch==="."||ch==="_"||ch==="-"||ch==="["||ch==="]"||ch==="*"||ch==="$";safe+=allowed?ch:"_"}if(safe.length<=MAX_MARKER_SOURCE_CHARS)return safe;let suffix=sha256Bytes(source).slice(0,12);return`${safe.slice(0,MAX_MARKER_SOURCE_CHARS-suffix.length-1)}~${suffix}`}function buildDataUriMarker(omission){let decoded=omission.decodedBytes===null?"unknown":String(omission.decodedBytes),decodeError=omission.decodeError?`; decodeError=${omission.decodeError}`:"";return`[pi-web-access inline data URI omitted; ordinal=${omission.ordinal}; source=${omission.sourcePath}; mime=${omission.mimeType}; encoding=${omission.encoding}; encodedBytes=${omission.encodedBytes}; decodedBytes=${decoded}${decodeError}; sha256=${omission.sha256}; digestBasis=${omission.digestBasis}; retrieval=not-retained]`}function sanitizeTextInternal(text,sourcePath,state){let scanFrom=0,retainedFrom=0,outputLength=0,pieces=[];for(;scanFrom<text.length;){let start=findNextDataScheme(text,scanFrom);if(start<0)break;let candidate=scanDataUriCandidate(text,start),end=candidate.end,headerStart=start+5,missingComma=candidate.comma<0;if(missingComma&&end===headerStart&&candidate.enclosure===null){scanFrom=end;continue}let headerEnd=missingComma?end:candidate.comma,headerLength=headerEnd-headerStart,encoding=headerEndsWithBase64(text,headerStart,headerEnd)?"base64":"percent-encoded",mimeType=normalizeMimeType(text,headerStart,headerEnd),payload=missingComma?text.slice(headerStart,end):text.slice(candidate.comma+1,end),encodedBytes=Buffer3.byteLength(payload,"utf8"),decodedBytes=null,decodeError,digestBasis="encoded",digest="";if(missingComma)decodeError="missing-comma",digest=sha256Bytes(payload);else if(headerLength>MAX_DATA_URI_HEADER_CHARS)decodeError="header-too-long",digest=sha256Bytes(payload);else{let decoded=encoding==="base64"?decodeBase64Payload(payload):decodePercentEncoded(payload);"error"in decoded?(decodeError=decoded.error,digest=sha256Bytes(payload)):(decodedBytes=decoded.bytes.length,digestBasis="decoded",digest=sha256Bytes(decoded.bytes))}let omission={ordinal:state.nextOrdinal++,sourcePath:safeMarkerSource(sourcePath),mimeType,encoding,encodedBytes,decodedBytes,...decodeError?{decodeError}:{},sha256:digest,digestBasis,retrieval:"not-retained"},marker=buildDataUriMarker(omission),prefix=text.slice(retainedFrom,start);pieces.push(prefix,marker),outputLength+=prefix.length;let markerStart=outputLength;outputLength+=marker.length,state.omissions.push({...omission,markerStart,markerEnd:outputLength}),retainedFrom=end,scanFrom=end}return state.omissions.length===0&&retainedFrom===0?text:(pieces.push(text.slice(retainedFrom)),pieces.join(""))}function sanitizeInlineDataUris(text,sourcePath){let state={nextOrdinal:1,omissions:[]};return{text:sanitizeTextInternal(text,sourcePath,state),omissions:state.omissions.map(({markerStart:_start,markerEnd:_end,...omission})=>omission)}}var DEFAULT_TIMEOUT_MS3=3e4,MAX_CONFIGURED_TIMEOUT_MS=2147483647,CONCURRENT_LIMIT=3,WEB_SEARCH_CONFIG_PATH=getWebSearchConfigPath();function loadFetchTimeoutMs(){if(!existsSync9(WEB_SEARCH_CONFIG_PATH))return DEFAULT_TIMEOUT_MS3;let raw;try{raw=JSON.parse(readFileSync9(WEB_SEARCH_CONFIG_PATH,"utf-8"))}catch(err){let message=err instanceof Error?err.message:String(err);throw new Error(`Failed to parse ${WEB_SEARCH_CONFIG_PATH}: ${message}`)}if(!raw||typeof raw!="object"||Array.isArray(raw))throw new Error(`Invalid config in ${WEB_SEARCH_CONFIG_PATH}: expected a JSON object`);let fetchConfig=raw.fetch;if(fetchConfig===void 0)return DEFAULT_TIMEOUT_MS3;if(!fetchConfig||typeof fetchConfig!="object"||Array.isArray(fetchConfig))throw new Error(`fetch in ${WEB_SEARCH_CONFIG_PATH} must be an object`);let value=fetchConfig.timeout;if(value===void 0)return DEFAULT_TIMEOUT_MS3;if(typeof value!="number"||!Number.isFinite(value)||value<=0)throw new Error(`Invalid fetch.timeout in ${WEB_SEARCH_CONFIG_PATH}: expected a positive finite number of seconds, got ${JSON.stringify(value)}`);let timeoutMs=Math.ceil(value*1e3);if(!Number.isSafeInteger(timeoutMs)||timeoutMs<1||timeoutMs>MAX_CONFIGURED_TIMEOUT_MS)throw new Error(`Invalid fetch.timeout in ${WEB_SEARCH_CONFIG_PATH}: converted timeout must be a finite safe integer from 1 through ${MAX_CONFIGURED_TIMEOUT_MS} milliseconds`);return Math.max(1,timeoutMs)}var NON_RECOVERABLE_ERRORS=["Unsupported content type","Response too large","PDF extraction is disabled","Image fetching is disabled"],MIN_USEFUL_CONTENT=500,SUPPORTED_IMAGE_TYPES=new Set(["image/png","image/jpeg","image/webp","image/gif"]),FETCH_PROVIDERS=["http","firecrawl","crawl4ai","jina","tinyfish","search1api","querit","kagi","ollama","parallel","parallel-mcp","brightdata","gemini"],DEFAULT_FETCH_PROVIDER_ORDER=["http","firecrawl","crawl4ai","jina","tinyfish","search1api","querit","kagi","ollama","parallel","brightdata","gemini"],REMOTE_HOSTED_FETCH_PROVIDERS=new Set(["jina","tinyfish","search1api","querit","kagi","ollama","parallel","parallel-mcp","brightdata","gemini"]);function isDefuddleConsoleError(args){let prefix=args[0];return prefix==="Defuddle"||typeof prefix=="string"&&/^Defuddle(?:\s|:)/.test(prefix)}async function extractWithDefuddle(text,url){let{Defuddle}=await import("./node-UMFM7CDZ.js"),{parseHTML}=await import("./esm-E7UVEYTO.js"),{document}=parseHTML(text);Object.defineProperty(document,"location",{value:new URL(url),configurable:!0});let processingError,originalConsoleError=console.error;console.error=(...args)=>{if(isDefuddleConsoleError(args)){args[0]==="Defuddle"&&args[1]==="Error processing document:"&&(processingError=args[2]);return}originalConsoleError(...args)};let resultPromise;try{resultPromise=Defuddle(document,url,{markdown:!0,useAsync:!1})}finally{console.error=originalConsoleError}let result=await resultPromise;if(processingError!==void 0)throw new Error(`Defuddle failed to process document: ${errorMessage5(processingError)}`);return typeof result.content=="string"?{title:result.title,content:result.content}:null}function loadSsrfAllowRanges(){return loadSsrfConfig().allowRanges}function errorMessage5(err){return err instanceof Error?err.message:String(err)}function isConfigParseError(err){return errorMessage5(err).startsWith("Failed to parse ")}function isAbortError3(err){return errorMessage5(err).toLowerCase().includes("abort")}function isAbortException2(err){return err instanceof DOMException&&(err.name==="AbortError"||err.name==="TimeoutError")}function isRedirectPolicyError(message){return message.startsWith("Authenticated fetch refused cross-origin redirect")||message.startsWith("Blocked internal ")||message.startsWith("Blocked hostname by fetch_content domain policy")||message.startsWith("Hostname not allowed by fetch_content domain policy")||message.startsWith("Too many redirects fetching ")||message==="Only HTTP and HTTPS URLs can be fetched remotely"||message==="URL must include a hostname"||message.startsWith("Failed to resolve ")}function imageGateError(){try{return isImageEnabled()?null:"Image fetching is disabled by image.enabled"}catch(err){return errorMessage5(err)}}async function resolveAuthCookieHeader(url,profile){let parsed=assertAuthFetchUrl(profile,url.toString()),result=await getBrowserCookiesForHosts({hosts:[parsed.hostname],profile:profile.chromeProfile,requestUrl:parsed});if(result?.cookieHeader)return result.cookieHeader;if(!result){let diagnostic=getLastBrowserCookieDiagnostic();throw new Error(`Authenticated fetch profile ${profile.name} could not read browser cookies${diagnostic?`: ${diagnostic}`:""}`)}throw new Error(`Authenticated fetch profile ${profile.name} could not build a cookie header`)}var REDIRECT_STATUSES2=new Set([301,302,303,307,308]);async function fetchAuthenticatedRemoteUrl(url,init,validationOptions,profile){let current=await validateRemoteUrl(url,{allowRanges:validationOptions.ssrf.allowRanges,trustEnvProxy:validationOptions.ssrf.trustEnvProxy,domainPolicy:validationOptions.domainPolicy,...validationOptions.lookup?{lookup:validationOptions.lookup}:{}}),requestInit=init;for(let redirects=0;redirects<=5;redirects++){let cookieHeader=await resolveAuthCookieHeader(current,profile),headers={...requestInit.headers,cookie:cookieHeader},response=await fetch(current,{...requestInit,headers,redirect:"manual"});if(!REDIRECT_STATUSES2.has(response.status))return response;let location=response.headers.get("location");if(!location)return response;if(redirects===5)throw new Error(`Too many redirects fetching ${current.toString()}`);let from=current;if(current=await validateRemoteUrl(new URL(location,current),{allowRanges:validationOptions.ssrf.allowRanges,trustEnvProxy:validationOptions.ssrf.trustEnvProxy,domainPolicy:validationOptions.domainPolicy,...validationOptions.lookup?{lookup:validationOptions.lookup}:{}}),authFetchRedirectGuard(profile,from,current),response.status===303||(response.status===301||response.status===302)&&requestInit.method?.toUpperCase()==="POST"){let{body:_body,...nextInit}=requestInit;requestInit={...nextInit,method:"GET"}}}throw new Error(`Too many redirects fetching ${current.toString()}`)}function loadFetchRouting(){if(!existsSync9(WEB_SEARCH_CONFIG_PATH))return{providers:DEFAULT_FETCH_PROVIDER_ORDER,allowRemoteHostedProviders:!1};let raw;try{let parsed=JSON.parse(readFileSync9(WEB_SEARCH_CONFIG_PATH,"utf-8"));if(!parsed||typeof parsed!="object"||Array.isArray(parsed))throw new Error("expected a JSON object");raw=parsed}catch(err){let message=err instanceof Error?err.message:String(err);throw new Error(`Failed to parse ${WEB_SEARCH_CONFIG_PATH}: ${message}`)}if(!Object.hasOwn(raw,"fetchRouting"))return{providers:DEFAULT_FETCH_PROVIDER_ORDER,allowRemoteHostedProviders:!1};let routing=raw.fetchRouting;if(!routing||typeof routing!="object"||Array.isArray(routing))throw new Error(`fetchRouting in ${WEB_SEARCH_CONFIG_PATH} must be an object`);let routingConfig=routing,providersValue=routingConfig.providers,providers=DEFAULT_FETCH_PROVIDER_ORDER;if(providersValue!==void 0){if(!Array.isArray(providersValue)||providersValue.length===0)throw new Error(`fetchRouting.providers in ${WEB_SEARCH_CONFIG_PATH} must be a non-empty array`);providers=[];for(let provider of providersValue){let normalized=typeof provider=="string"?provider.trim().toLowerCase():"";if(!FETCH_PROVIDERS.includes(normalized))throw new Error(`fetchRouting.providers in ${WEB_SEARCH_CONFIG_PATH} contains an invalid provider: ${String(provider)}`);if(providers.includes(normalized))throw new Error(`fetchRouting.providers in ${WEB_SEARCH_CONFIG_PATH} must not contain duplicates: ${normalized}`);providers.push(normalized)}}let allowRemoteHostedProvidersValue=routingConfig.allowRemoteHostedProviders;if(allowRemoteHostedProvidersValue!==void 0&&typeof allowRemoteHostedProvidersValue!="boolean")throw new Error(`fetchRouting.allowRemoteHostedProviders in ${WEB_SEARCH_CONFIG_PATH} must be a boolean`);return{providers,allowRemoteHostedProviders:allowRemoteHostedProvidersValue===!0}}function notFoundGuidance(result,toolNames){let lines=[result.error??`HTTP ${result.status}`,"",`The origin server says this page does not exist (HTTP ${result.status}), so extraction providers cannot retrieve it.`];return toolNames?.webSearch&&toolNames.fetchContent?lines.push(`The page may have moved or been renamed. Use ${toolNames.webSearch} to find the current URL, then retry ${toolNames.fetchContent} with it.`):toolNames?.webSearch?lines.push(`The page may have moved or been renamed. Use ${toolNames.webSearch} to find the current URL.`):lines.push("The page may have moved or been renamed. Find the current URL, then retry the fetch with it."),lines.join(`
+`)}function abortedResult(url){return{url,title:"",content:"",error:"Aborted"}}var turndownInstance;async function loadTurndown(){let{default:TurndownService}=await import("./turndown.cjs-HRMZVRUL.js");return new TurndownService({headingStyle:"atx",codeBlockStyle:"fenced"})}function getTurndown(){return turndownInstance??=loadTurndown(),turndownInstance}var fetchLimit=pLimit(CONCURRENT_LIMIT);function resolveFetchTimeoutMs(options){return options?.timeoutMs??loadFetchTimeoutMs()}var JINA_READER_BASE="https://r.jina.ai/";async function extractWithJinaReader(url,timeoutMs,signal,lookup){let jinaUrl=JINA_READER_BASE+url,activityId=activityMonitor.logStart({type:"api",query:`jina: ${url}`});try{let ssrf=loadSsrfConfig(),domainPolicy=loadFetchContentDomainPolicy();await validateRemoteUrl(url,{allowRanges:ssrf.allowRanges,trustEnvProxy:ssrf.trustEnvProxy,domainPolicy,...lookup?{lookup}:{}});let res=await fetch(jinaUrl,{headers:{Accept:"text/markdown","X-No-Cache":"true"},signal:AbortSignal.any([AbortSignal.timeout(timeoutMs),...signal?[signal]:[]])});if(!res.ok)return activityMonitor.logComplete(activityId,res.status),null;let content=await res.text();activityMonitor.logComplete(activityId,res.status);let contentStart=content.indexOf("Markdown Content:");if(contentStart<0)return null;let markdownPart=content.slice(contentStart+17).trim();if(markdownPart.length<100||markdownPart.startsWith("Loading...")||markdownPart.startsWith("Please enable JavaScript"))return null;let title=extractHeadingTitle(markdownPart)??(new URL(url).pathname.split("/").pop()||url);return{url,title,content:markdownPart,error:null}}catch(err){let message=err instanceof Error?err.message:String(err);return message.toLowerCase().includes("abort")?activityMonitor.logComplete(activityId,0):activityMonitor.logError(activityId,message),null}}function parseTimestamp(ts){let num=Number(ts);if(!isNaN(num)&&num>=0)return Math.floor(num);let parts=ts.split(":").map(Number);return parts.some(p=>isNaN(p)||p<0)?null:parts.length===3?Math.floor(parts[0]*3600+parts[1]*60+parts[2]):parts.length===2?Math.floor(parts[0]*60+parts[1]):null}function parseTimestampSpec(ts){let dashIdx=ts.indexOf("-",1);if(dashIdx>0){let start=parseTimestamp(ts.slice(0,dashIdx)),end=parseTimestamp(ts.slice(dashIdx+1));if(start!==null&&end!==null&&end>start)return{type:"range",start,end}}let seconds=parseTimestamp(ts);return seconds!==null?{type:"single",seconds}:null}var DEFAULT_RANGE_FRAMES=6,MIN_FRAME_INTERVAL=5;function computeRangeTimestamps(start,end,maxFrames=DEFAULT_RANGE_FRAMES){if(maxFrames<=1)return[start];let idealInterval=(end-start)/(maxFrames-1);if(idealInterval<MIN_FRAME_INTERVAL){let timestamps=[];for(let t=start;t<=end&&timestamps.length<maxFrames;t+=MIN_FRAME_INTERVAL)timestamps.push(t);return timestamps}return Array.from({length:maxFrames},(_,i)=>Math.round(start+i*idealInterval))}function buildFrameResult(url,label,requestedCount,frames,error,duration){if(frames.length===0){let msg=error??"Frame extraction failed";return{url,title:`Frames ${label} (0/${requestedCount})`,content:msg,error:msg}}return{url,title:`Frames ${label} (${frames.length}/${requestedCount})`,content:`${frames.length} frames extracted from ${label}`,error:null,frames,...duration!==void 0?{duration}:{}}}async function extractLocalFrames(filePath,timestamps){let results=await Promise.all(timestamps.map(async t=>{let frame=await extractVideoFrame(filePath,t);return"error"in frame?{error:frame.error}:{...frame,timestamp:formatSeconds(t)}})),frames=results.filter(f=>"data"in f),firstError=results.find(f=>"error"in f);return{frames,error:frames.length===0&&firstError?firstError.error:null}}function safeVideoInfo(url){try{let info=isVideoFile(url);return info?{status:"video",info}:{status:"not-video"}}catch(err){return{status:"invalid",error:errorMessage5(err)}}}async function extractContent(url,signal,options){if(signal?.aborted)return{url,title:"",content:"",error:"Aborted"};let remoteUrl=null;try{let parsed=new URL(url);(parsed.protocol==="http:"||parsed.protocol==="https:")&&(remoteUrl=parsed)}catch{}if(remoteUrl)try{let ssrf=loadSsrfConfig(),domainPolicy=loadFetchContentDomainPolicy();await validateRemoteUrl(remoteUrl,{allowRanges:ssrf.allowRanges,trustEnvProxy:ssrf.trustEnvProxy,domainPolicy,...options?.lookup?{lookup:options.lookup}:{}})}catch(err){return{url,title:"",content:"",error:errorMessage5(err)}}if(options?.authFetchProfile||options?.mode==="raw")try{return await extractViaHttp(url,resolveFetchTimeoutMs(options),signal,options)}catch(err){return{url,title:"",content:"",error:errorMessage5(err)}}if(options?.frames||options?.timestamp){let disabled=imageGateError();if(disabled)return{url,title:"",content:"",error:disabled}}if(options?.frames&&!options.timestamp){let frameCount=options.frames,ytInfo2=isYouTubeURL(url);if(ytInfo2.isYouTube&&ytInfo2.videoId){let streamInfo=await getYouTubeStreamInfo(ytInfo2.videoId);if("error"in streamInfo)return{url,title:"Frames",content:streamInfo.error,error:streamInfo.error};if(streamInfo.duration===null){let error="Cannot determine video duration. Use a timestamp range instead.";return{url,title:"Frames",content:error,error}}let dur=Math.floor(streamInfo.duration),timestamps=computeRangeTimestamps(0,dur,frameCount),result=await extractYouTubeFrames(ytInfo2.videoId,timestamps,streamInfo),label=`${formatSeconds(0)}-${formatSeconds(dur)}`;return buildFrameResult(url,label,timestamps.length,result.frames,result.error,streamInfo.duration)}let localVideo2=safeVideoInfo(url);if(localVideo2.status==="invalid")return{url,title:"",content:"",error:localVideo2.error};if(localVideo2.status==="video"){let durationResult=await getLocalVideoDuration(localVideo2.info.absolutePath);if(typeof durationResult!="number")return{url,title:"Frames",content:durationResult.error,error:durationResult.error};let dur=Math.floor(durationResult),timestamps=computeRangeTimestamps(0,dur,frameCount),result=await extractLocalFrames(localVideo2.info.absolutePath,timestamps),label=`${formatSeconds(0)}-${formatSeconds(dur)}`;return buildFrameResult(url,label,timestamps.length,result.frames,result.error,durationResult)}return{url,title:"",content:"",error:"Frame extraction only works with YouTube and local video files"}}if(options?.timestamp){let spec=parseTimestampSpec(options.timestamp);if(!spec)return{url,title:"",content:"",error:`Invalid timestamp format: "${options.timestamp}". Use "H:MM:SS", "MM:SS", "85", or "start-end".`};let frameCount=options.frames,ytInfo2=isYouTubeURL(url);if(ytInfo2.isYouTube&&ytInfo2.videoId){let streamInfo=await getYouTubeStreamInfo(ytInfo2.videoId);if("error"in streamInfo){if(spec.type==="range"){let label=`${formatSeconds(spec.start)}-${formatSeconds(spec.end)}`;return{url,title:`Frames ${label}`,content:streamInfo.error,error:streamInfo.error}}if(frameCount){let end=spec.seconds+(frameCount-1)*MIN_FRAME_INTERVAL,label=`${formatSeconds(spec.seconds)}-${formatSeconds(end)}`;return{url,title:`Frames ${label}`,content:streamInfo.error,error:streamInfo.error}}return{url,title:`Frame at ${options.timestamp}`,content:streamInfo.error,error:streamInfo.error}}if(spec.type==="range"){let label=`${formatSeconds(spec.start)}-${formatSeconds(spec.end)}`;if(streamInfo.duration!==null&&spec.end>streamInfo.duration){let error=`Timestamp ${formatSeconds(spec.end)} exceeds video duration (${formatSeconds(Math.floor(streamInfo.duration))})`;return{url,title:`Frames ${label}`,content:error,error}}let timestamps=frameCount?computeRangeTimestamps(spec.start,spec.end,frameCount):computeRangeTimestamps(spec.start,spec.end),result=await extractYouTubeFrames(ytInfo2.videoId,timestamps,streamInfo);return buildFrameResult(url,label,timestamps.length,result.frames,result.error,result.duration??void 0)}if(frameCount){let end=spec.seconds+(frameCount-1)*MIN_FRAME_INTERVAL,label=`${formatSeconds(spec.seconds)}-${formatSeconds(end)}`;if(streamInfo.duration!==null&&end>streamInfo.duration){let error=`Timestamp ${formatSeconds(end)} exceeds video duration (${formatSeconds(Math.floor(streamInfo.duration))})`;return{url,title:`Frames ${label}`,content:error,error}}let timestamps=computeRangeTimestamps(spec.seconds,end,frameCount),result=await extractYouTubeFrames(ytInfo2.videoId,timestamps,streamInfo);return buildFrameResult(url,label,timestamps.length,result.frames,result.error,result.duration??void 0)}if(streamInfo.duration!==null&&spec.seconds>streamInfo.duration){let error=`Timestamp ${formatSeconds(spec.seconds)} exceeds video duration (${formatSeconds(Math.floor(streamInfo.duration))})`;return{url,title:`Frame at ${options.timestamp}`,content:error,error}}let frame=await extractYouTubeFrame(ytInfo2.videoId,spec.seconds,streamInfo);return"error"in frame?{url,title:`Frame at ${options.timestamp}`,content:frame.error,error:frame.error}:{url,title:`Frame at ${options.timestamp}`,content:`Video frame at ${options.timestamp}`,error:null,thumbnail:frame}}let localVideo2=safeVideoInfo(url);if(localVideo2.status==="invalid")return{url,title:"",content:"",error:localVideo2.error};if(localVideo2.status==="video"){if(spec.type==="range"){let timestamps=frameCount?computeRangeTimestamps(spec.start,spec.end,frameCount):computeRangeTimestamps(spec.start,spec.end),result=await extractLocalFrames(localVideo2.info.absolutePath,timestamps),label=`${formatSeconds(spec.start)}-${formatSeconds(spec.end)}`;return buildFrameResult(url,label,timestamps.length,result.frames,result.error)}if(frameCount){let end=spec.seconds+(frameCount-1)*MIN_FRAME_INTERVAL,timestamps=computeRangeTimestamps(spec.seconds,end,frameCount),result=await extractLocalFrames(localVideo2.info.absolutePath,timestamps),label=`${formatSeconds(spec.seconds)}-${formatSeconds(end)}`;return buildFrameResult(url,label,timestamps.length,result.frames,result.error)}let frame=await extractVideoFrame(localVideo2.info.absolutePath,spec.seconds);return"error"in frame?{url,title:`Frame at ${options.timestamp}`,content:frame.error,error:frame.error}:{url,title:`Frame at ${options.timestamp}`,content:`Video frame at ${options.timestamp}`,error:null,thumbnail:frame}}return{url,title:"",content:"",error:"Timestamp extraction only works with YouTube and local video files"}}let localVideo=safeVideoInfo(url);if(localVideo.status==="invalid")return{url,title:"",content:"",error:localVideo.error};if(localVideo.status==="video")try{let result=await extractVideo(localVideo.info,signal,options);return signal?.aborted?abortedResult(url):result??{url,title:"",content:"",error:`Video analysis requires Gemini access. Either:
   1. Sign into gemini.google.com in Chrome (free, uses cookies)
-  2. Set GEMINI_API_KEY in ${WEB_SEARCH_CONFIG_PATH}`}}catch(err){return isAbortError3(err)?abortedResult(url):{url,title:"",content:"",error:errorMessage4(err)}}try{remoteUrl||new URL(url)}catch(err){return{url,title:"",content:"",error:errorMessage4(err)}}try{let ghIssuePrResult=await extractGitHubIssuePr(url,signal,options);if(ghIssuePrResult)return ghIssuePrResult;if(signal?.aborted)return abortedResult(url)}catch(err){let message=errorMessage4(err);if(isAbortError3(err))return abortedResult(url);if(isConfigParseError(err))return{url,title:"",content:"",error:message}}try{let ghResult=await extractGitHub(url,signal,options?.forceClone);if(ghResult)return ghResult;if(signal?.aborted)return abortedResult(url)}catch(err){let message=errorMessage4(err);if(isAbortError3(err))return abortedResult(url);if(isConfigParseError(err))return{url,title:"",content:"",error:message}}let ytInfo=isYouTubeURL(url),youtubeEnabled=!1;try{youtubeEnabled=isYouTubeEnabled()}catch(err){return{url,title:"",content:"",error:errorMessage4(err)}}if(ytInfo.isYouTube&&youtubeEnabled){try{let ytResult=await extractYouTube(url,signal,options?.prompt,options?.model);if(ytResult)return ytResult;if(signal?.aborted)return abortedResult(url)}catch(err){let message=errorMessage4(err);return isAbortError3(err)?abortedResult(url):{url,title:"",content:"",error:message}}return{url,title:"",content:"",error:"Could not extract YouTube video content. Sign into Google in a supported Chromium browser for automatic access, or set GEMINI_API_KEY."}}if(signal?.aborted)return abortedResult(url);let fetchTimeoutMs;try{fetchTimeoutMs=resolveFetchTimeoutMs(options)}catch(err){return{url,title:"",content:"",error:errorMessage4(err)}}let fetchRouting;try{fetchRouting=loadFetchRouting()}catch(err){return{url,title:"",content:"",error:errorMessage4(err)}}let providerOrder=remoteUrl&&!fetchRouting.allowRemoteHostedProviders?fetchRouting.providers.filter(provider=>!REMOTE_HOSTED_FETCH_PROVIDERS.has(provider)):fetchRouting.providers;if(providerOrder.length===0)return{url,title:"",content:"",error:"Remote hosted fetch providers are disabled unless fetchRouting.allowRemoteHostedProviders is true"};let httpResult=null,declaredLinks=[],withDeclaredLinks=result=>({...result,content:appendDeclaredWebLinks(result.content,declaredLinks)}),parseErrorResult=message=>httpResult?{...httpResult,error:message}:{url,title:"",content:"",error:message},runHttpProvider=async()=>{let{declaredLinks:discoveredLinks=[],...result}=await extractViaHttp(url,fetchTimeoutMs,signal,options);return httpResult=result,declaredLinks=discoveredLinks,signal?.aborted?abortedResult(url):!httpResult.error||NON_RECOVERABLE_ERRORS.some(prefix=>httpResult.error.startsWith(prefix))||isRedirectPolicyError(httpResult.error)||isConfigParseError(httpResult.error)?httpResult:null},firecrawlError=null,tinyfishError=null,search1apiError=null,queritError=null,kagiError=null,ollamaError=null,parallelError=null,parallelMcpError=null,brightdataError=null;if(remoteUrl&&providerOrder[0]!=="http"){let httpGateResult=await runHttpProvider();if(httpGateResult)return httpGateResult}for(let provider of providerOrder){if(signal?.aborted)return abortedResult(url);if(provider==="http"){let result=await runHttpProvider();if(result)return result;continue}if(provider==="firecrawl"){try{if(isFirecrawlAvailable()){let ssrf=loadSsrfConfig(),firecrawlResult=await extractWithFirecrawl(url,signal,{timeoutMs:options?.timeoutMs,...options?.lookup?{lookup:options.lookup}:{},ssrf});if(firecrawlResult)return withDeclaredLinks(firecrawlResult)}}catch(err){if(isAbortError3(err))return abortedResult(url);if(firecrawlError=errorMessage4(err),isConfigParseError(err))return parseErrorResult(firecrawlError)}continue}if(provider==="jina"){let jinaResult=await extractWithJinaReader(url,fetchTimeoutMs,signal,options?.lookup);if(jinaResult)return withDeclaredLinks(jinaResult);continue}if(provider==="tinyfish"){try{if(isTinyFishAvailable()){let tinyfishResult=await extractWithTinyFish(url,signal,options);if(tinyfishResult)return withDeclaredLinks(tinyfishResult)}}catch(err){if(isAbortError3(err))return abortedResult(url);if(tinyfishError=errorMessage4(err),isConfigParseError(err))return parseErrorResult(tinyfishError)}continue}if(provider==="search1api"){try{if(isSearch1APIAvailable()){let search1apiResult=await extractWithSearch1API(url,signal,options);if(search1apiResult)return withDeclaredLinks(search1apiResult)}}catch(err){if(isAbortError3(err))return abortedResult(url);if(search1apiError=errorMessage4(err),isConfigParseError(err))return parseErrorResult(search1apiError)}continue}if(provider==="querit"){try{if(isQueritAvailable()){let queritResult=await extractWithQuerit(url,signal,options);if(queritResult)return withDeclaredLinks(queritResult)}}catch(err){if(isAbortError3(err))return abortedResult(url);if(queritError=errorMessage4(err),isConfigParseError(err))return parseErrorResult(queritError)}continue}if(provider==="kagi"){try{if(isKagiExtractAvailable()){let ssrf=loadSsrfConfig(),kagiResult=await extractWithKagi(url,signal,{timeoutMs:options?.timeoutMs,...options?.lookup?{lookup:options.lookup}:{},ssrf});if(kagiResult)return withDeclaredLinks(kagiResult)}}catch(err){if(isAbortError3(err))return abortedResult(url);if(kagiError=errorMessage4(err),isConfigParseError(err))return parseErrorResult(kagiError)}continue}if(provider==="ollama"){try{if(isOllamaFetchAvailable()){let ssrf=loadSsrfConfig(),ollamaResult=await extractWithOllama(url,signal,{timeoutMs:options?.timeoutMs,...options?.lookup?{lookup:options.lookup}:{},ssrf});if(ollamaResult)return withDeclaredLinks(ollamaResult)}}catch(err){if(isAbortError3(err))return abortedResult(url);if(ollamaError=errorMessage4(err),isConfigParseError(err))return parseErrorResult(ollamaError)}continue}if(provider==="parallel"){try{if(isParallelAvailable()){let parallelResult=await extractWithParallel(url,signal,options);if(parallelResult)return withDeclaredLinks(parallelResult)}}catch(err){if(isAbortError3(err))return abortedResult(url);if(parallelError=errorMessage4(err),isConfigParseError(err))return parseErrorResult(parallelError)}continue}if(provider==="parallel-mcp"){try{let parallelMcpResult=await extractWithParallelMcp(url,signal,options);if(parallelMcpResult)return withDeclaredLinks(parallelMcpResult)}catch(err){if(isAbortError3(err))return abortedResult(url);if(parallelMcpError=errorMessage4(err),isConfigParseError(err))return parseErrorResult(parallelMcpError)}continue}if(provider==="brightdata"){try{if(isBrightDataUnlockerAvailable()){let ssrf=loadSsrfConfig(),brightdataResult=await extractWithBrightDataUnlocker(url,signal,{timeoutMs:options?.timeoutMs,...options?.lookup?{lookup:options.lookup}:{},ssrf});if(brightdataResult)return withDeclaredLinks(brightdataResult)}}catch(err){if(isAbortError3(err))return abortedResult(url);if(brightdataError=errorMessage4(err),isConfigParseError(err))return parseErrorResult(brightdataError)}continue}if(provider==="gemini"){let geminiResult=null;try{geminiResult=await extractWithUrlContext(url,signal)??await extractWithGeminiWeb(url,signal)}catch(err){if(isAbortError3(err))return abortedResult(url);if(err instanceof CredentialResolutionError||isConfigParseError(err))return parseErrorResult(errorMessage4(err))}if(geminiResult)return withDeclaredLinks(geminiResult)}}if(signal?.aborted)return abortedResult(url);let finalHttpResult=httpResult;if(finalHttpResult&&declaredLinks.length>0)return{...finalHttpResult,error:null};if(finalHttpResult?.status===404||finalHttpResult?.status===410)return{...finalHttpResult,error:notFoundGuidance(finalHttpResult,options?.toolNames)};let searchToolName=options?.toolNames?.webSearch,guidance=[finalHttpResult?.error??"No fetch_content provider returned content",...firecrawlError?[`Firecrawl fallback failed: ${firecrawlError}`]:[],...tinyfishError?[`TinyFish fallback failed: ${tinyfishError}`]:[],...search1apiError?[`Search1API fallback failed: ${search1apiError}`]:[],...queritError?[`Querit fallback failed: ${queritError}`]:[],...kagiError?[`Kagi fallback failed: ${kagiError}`]:[],...ollamaError?[`Ollama fallback failed: ${ollamaError}`]:[],...parallelError?[`Parallel fallback failed: ${parallelError}`]:[],...parallelMcpError?[`Parallel MCP fallback failed: ${parallelMcpError}`]:[],...brightdataError?[`Bright Data fallback failed: ${brightdataError}`]:[],"","Fallback options:",`  \u2022 Set firecrawlBaseUrl in ${WEB_SEARCH_CONFIG_PATH} to a self-hosted Firecrawl instance`,`  \u2022 Set tinyfishApiKey in ${WEB_SEARCH_CONFIG_PATH} or TINYFISH_API_KEY`,`  \u2022 Set search1apiApiKey in ${WEB_SEARCH_CONFIG_PATH} or SEARCH1API_KEY`,`  \u2022 Set queritApiKey in ${WEB_SEARCH_CONFIG_PATH} or QUERIT_API_KEY`,`  \u2022 Set kagiApiKey in ${WEB_SEARCH_CONFIG_PATH} or KAGI_API_KEY`,`  \u2022 Set ollamaApiKey in ${WEB_SEARCH_CONFIG_PATH} or OLLAMA_API_KEY`,`  \u2022 Set parallelApiKey in ${WEB_SEARCH_CONFIG_PATH} or PARALLEL_API_KEY`,`  \u2022 Set brightdataApiKey and brightdataUnlockerZone in ${WEB_SEARCH_CONFIG_PATH} or BRIGHTDATA_API_KEY and BRIGHTDATA_UNLOCKER_ZONE`,`  \u2022 Set GEMINI_API_KEY in ${WEB_SEARCH_CONFIG_PATH}`,"  \u2022 Sign into gemini.google.com in Chrome",...searchToolName?[`  \u2022 Use ${searchToolName} to find content about this topic`]:[]].join(`
-`);return{...finalHttpResult??{url,title:"",content:"",error:null},error:guidance}}function isLikelyJSRendered(html){let bodyMatch=html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);if(!bodyMatch)return!1;let textContent=bodyMatch[1].replace(/<script[\s\S]*?<\/script>/gi,"").replace(/<style[\s\S]*?<\/style>/gi,"").replace(/<[^>]+>/g,"").replace(/\s+/g," ").trim(),scriptCount=(html.match(/<script/gi)||[]).length;return textContent.length<500&&scriptCount>3}async function readPDFResponseBuffer(response,maxSizeMB){let maxBytes=maxSizeMB*1024*1024;return readResponseBufferWithLimit(response,maxBytes,()=>pdfSizeLimitError(maxSizeMB))}async function readTextResponseWithLimit(response,maxBytes){let buffer=await readResponseBufferWithLimit(response,maxBytes,()=>responseSizeLimitError(maxBytes)),charset=response.headers.get("content-type")?.match(/charset\s*=\s*["']?([^;"'\s]+)/i)?.[1];try{return new TextDecoder(charset||"utf-8").decode(buffer)}catch{return new TextDecoder("utf-8").decode(buffer)}}function isTextContentType(contentType){let mimeType=contentType.split(";",1)[0]?.trim().toLowerCase()??"";return mimeType.startsWith("text/")||mimeType==="application/json"||mimeType==="application/ld+json"||mimeType==="application/xml"||mimeType==="application/xhtml+xml"||mimeType==="application/javascript"||mimeType==="application/x-javascript"||mimeType.endsWith("+json")||mimeType.endsWith("+xml")}async function readResponseBufferWithLimit(response,maxBytes,buildError){let reader=response.body?.getReader();if(!reader){let buffer=await response.arrayBuffer();if(buffer.byteLength>maxBytes)throw buildError();return buffer}let chunks=[],totalBytes=0;try{for(;;){let{done,value}=await reader.read();if(done)break;if(value){if(totalBytes+=value.byteLength,totalBytes>maxBytes)throw await reader.cancel(),buildError();chunks.push(value)}}}finally{reader.releaseLock()}let combined=new Uint8Array(totalBytes),offset=0;for(let chunk of chunks)combined.set(chunk,offset),offset+=chunk.byteLength;return combined.buffer}function pdfSizeLimitError(maxSizeMB){return new Error(`PDF exceeds configured pdf.maxSizeMB limit (${maxSizeMB} MB)`)}function responseSizeLimitError(maxBytes){return new Error(`Response too large (${Math.round(maxBytes/1024/1024)}MB)`)}async function extractViaHttp(url,timeoutMs,signal,options){let activityId=activityMonitor.logStart({type:"fetch",url}),controller=new AbortController,timeoutId=setTimeout(()=>controller.abort(),timeoutMs),onAbort=()=>controller.abort();signal?.addEventListener("abort",onAbort);try{let ssrf=loadSsrfConfig(),domainPolicy=loadFetchContentDomainPolicy(),authProfile=options?.authFetchProfile,trustEnvProxy=options?.proxy===void 0&&ssrf.trustEnvProxy,requestInit={signal:controller.signal,__proxy:options?.proxy,headers:{"User-Agent":"OpenAI File Downloader, XaiImageApiFetch/1.0",Accept:"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8","Accept-Language":"en-US,en;q=0.9","Cache-Control":"no-cache","Sec-Fetch-Dest":"document","Sec-Fetch-Mode":"navigate","Sec-Fetch-Site":"none","Sec-Fetch-User":"?1","Upgrade-Insecure-Requests":"1"}},response=authProfile?await fetchAuthenticatedRemoteUrl(url,requestInit,{ssrf:{...ssrf,trustEnvProxy},domainPolicy,...options?.lookup?{lookup:options.lookup}:{}},authProfile):await fetchRemoteUrl(url,requestInit,{allowRanges:ssrf.allowRanges,trustEnvProxy,domainPolicy,...options?.lookup?{lookup:options.lookup}:{}});if(!response.ok&&options?.mode!=="raw")return activityMonitor.logComplete(activityId,response.status),{url,title:"",content:"",error:`HTTP ${response.status}: ${response.statusText}`,status:response.status};let contentLengthHeader=response.headers.get("content-length"),contentType=response.headers.get("content-type")||"",mimeType=contentType.split(";",1)[0]?.trim().toLowerCase()??"",isPDFContent=isPDF(url,contentType),pdfConfig=isPDFContent?loadPDFConfig():null;if(isPDFContent&&pdfConfig&&!pdfConfig.enabled)return activityMonitor.logComplete(activityId,response.status),{url,title:"",content:"",error:"PDF extraction is disabled by pdf.enabled",mimeType,status:response.status};let maxResponseSize=(pdfConfig?.maxSizeMB??5)*1024*1024;if(contentLengthHeader){let contentLength=Number.parseInt(contentLengthHeader,10);if(Number.isFinite(contentLength)&&contentLength>maxResponseSize)return activityMonitor.logComplete(activityId,response.status),{url,title:"",content:"",error:pdfConfig?pdfSizeLimitError(pdfConfig.maxSizeMB).message:`Response too large (${Math.round(contentLength/1024/1024)}MB)`}}if(options?.mode==="raw"){if(!isTextContentType(contentType))return activityMonitor.logComplete(activityId,response.status),{url,title:"",content:"",error:`Unsupported content type in raw mode: ${mimeType||"missing"}`,mimeType,status:response.status};let text2=await readTextResponseWithLimit(response,maxResponseSize);return activityMonitor.logComplete(activityId,response.status),{url,title:extractTextTitle(text2,url),content:text2,error:null,mimeType,status:response.status}}if(SUPPORTED_IMAGE_TYPES.has(mimeType)){let disabled=imageGateError();if(disabled)return activityMonitor.logComplete(activityId,response.status),{url,title:"",content:"",error:disabled,mimeType,status:response.status};try{let buffer=await readResponseBufferWithLimit(response,maxResponseSize,()=>responseSizeLimitError(maxResponseSize)),resized=await resizeImage(new Uint8Array(buffer),mimeType,{maxWidth:2e3,maxHeight:2e3});if(activityMonitor.logComplete(activityId,response.status),!resized)return{url,title:"",content:"",error:`Could not decode image: ${mimeType}`,mimeType,status:response.status};let title=new URL(response.url||url).pathname.split("/").pop()||url;return{url,title,content:`Image fetched (${resized.width}\xD7${resized.height}, ${resized.mimeType})`,error:null,thumbnail:{data:resized.data,mimeType:resized.mimeType},mimeType:resized.mimeType,status:response.status}}catch(err){let message=errorMessage4(err);return activityMonitor.logError(activityId,message),{url,title:"",content:"",error:message,mimeType,status:response.status}}}if(isPDFContent&&pdfConfig)try{let buffer=await readPDFResponseBuffer(response,pdfConfig.maxSizeMB);if(signal?.aborted)return abortedResult(url);let result=await extractPDFToMarkdown(buffer,url,{signal});return activityMonitor.logComplete(activityId,response.status),{url,title:result.title,content:`PDF extracted and saved to: ${result.outputPath}
+  2. Set GEMINI_API_KEY in ${WEB_SEARCH_CONFIG_PATH}`}}catch(err){return isAbortError3(err)?abortedResult(url):{url,title:"",content:"",error:errorMessage5(err)}}try{remoteUrl||new URL(url)}catch(err){return{url,title:"",content:"",error:errorMessage5(err)}}try{let ghIssuePrResult=await extractGitHubIssuePr(url,signal,options);if(ghIssuePrResult)return ghIssuePrResult;if(signal?.aborted)return abortedResult(url)}catch(err){let message=errorMessage5(err);if(isAbortError3(err))return abortedResult(url);if(isConfigParseError(err))return{url,title:"",content:"",error:message}}try{let ghResult=await extractGitHub(url,signal,options?.forceClone);if(ghResult)return ghResult;if(signal?.aborted)return abortedResult(url)}catch(err){let message=errorMessage5(err);if(isAbortError3(err))return abortedResult(url);if(isConfigParseError(err))return{url,title:"",content:"",error:message}}let ytInfo=isYouTubeURL(url),youtubeEnabled=!1;try{youtubeEnabled=isYouTubeEnabled()}catch(err){return{url,title:"",content:"",error:errorMessage5(err)}}if(ytInfo.isYouTube&&youtubeEnabled){try{let ytResult=await extractYouTube(url,signal,options?.prompt,options?.model);if(ytResult)return ytResult;if(signal?.aborted)return abortedResult(url)}catch(err){let message=errorMessage5(err);return isAbortError3(err)?abortedResult(url):{url,title:"",content:"",error:message}}return{url,title:"",content:"",error:"Could not extract YouTube video content. Sign into Google in a supported Chromium browser for automatic access, or set GEMINI_API_KEY."}}if(signal?.aborted)return abortedResult(url);let fetchTimeoutMs;try{fetchTimeoutMs=resolveFetchTimeoutMs(options)}catch(err){return{url,title:"",content:"",error:errorMessage5(err)}}let fetchRouting;try{fetchRouting=loadFetchRouting()}catch(err){return{url,title:"",content:"",error:errorMessage5(err)}}let providerOrder=remoteUrl&&!fetchRouting.allowRemoteHostedProviders?fetchRouting.providers.filter(provider=>!REMOTE_HOSTED_FETCH_PROVIDERS.has(provider)):fetchRouting.providers;if(providerOrder.length===0)return{url,title:"",content:"",error:"Remote hosted fetch providers are disabled unless fetchRouting.allowRemoteHostedProviders is true"};let httpResult=null,declaredLinks=[],withDeclaredLinks=result=>({...result,content:appendDeclaredWebLinks(result.content,declaredLinks)}),parseErrorResult=message=>httpResult?{...httpResult,error:message}:{url,title:"",content:"",error:message},runHttpProvider=async()=>{let{declaredLinks:discoveredLinks=[],...result}=await extractViaHttp(url,fetchTimeoutMs,signal,options);return httpResult=result,declaredLinks=discoveredLinks,signal?.aborted?abortedResult(url):!httpResult.error||NON_RECOVERABLE_ERRORS.some(prefix=>httpResult.error.startsWith(prefix))||isRedirectPolicyError(httpResult.error)||isConfigParseError(httpResult.error)?httpResult:null},firecrawlError=null,crawl4aiError=null,tinyfishError=null,search1apiError=null,queritError=null,kagiError=null,ollamaError=null,parallelError=null,parallelMcpError=null,brightdataError=null;if(remoteUrl&&providerOrder[0]!=="http"){let httpGateResult=await runHttpProvider();if(httpGateResult)return httpGateResult}for(let provider of providerOrder){if(signal?.aborted)return abortedResult(url);if(provider==="http"){let result=await runHttpProvider();if(result)return result;continue}if(provider==="firecrawl"){try{if(isFirecrawlAvailable()){let ssrf=loadSsrfConfig(),firecrawlResult=await extractWithFirecrawl(url,signal,{timeoutMs:options?.timeoutMs,...options?.lookup?{lookup:options.lookup}:{},ssrf});if(firecrawlResult)return withDeclaredLinks(firecrawlResult)}}catch(err){if(isAbortError3(err))return abortedResult(url);if(firecrawlError=errorMessage5(err),isConfigParseError(err))return parseErrorResult(firecrawlError)}continue}if(provider==="crawl4ai"){try{if(isCrawl4aiAvailable()){let ssrf=loadSsrfConfig(),crawl4aiResult=await extractWithCrawl4ai(url,signal,{timeoutMs:options?.timeoutMs,...options?.lookup?{lookup:options.lookup}:{},ssrf});if(crawl4aiResult)return withDeclaredLinks(crawl4aiResult)}}catch(err){if(signal?.aborted||isAbortException2(err))return abortedResult(url);if(crawl4aiError=errorMessage5(err),isConfigParseError(err))return parseErrorResult(crawl4aiError)}continue}if(provider==="jina"){let jinaResult=await extractWithJinaReader(url,fetchTimeoutMs,signal,options?.lookup);if(jinaResult)return withDeclaredLinks(jinaResult);continue}if(provider==="tinyfish"){try{if(isTinyFishAvailable()){let tinyfishResult=await extractWithTinyFish(url,signal,options);if(tinyfishResult)return withDeclaredLinks(tinyfishResult)}}catch(err){if(isAbortError3(err))return abortedResult(url);if(tinyfishError=errorMessage5(err),isConfigParseError(err))return parseErrorResult(tinyfishError)}continue}if(provider==="search1api"){try{if(isSearch1APIAvailable()){let search1apiResult=await extractWithSearch1API(url,signal,options);if(search1apiResult)return withDeclaredLinks(search1apiResult)}}catch(err){if(isAbortError3(err))return abortedResult(url);if(search1apiError=errorMessage5(err),isConfigParseError(err))return parseErrorResult(search1apiError)}continue}if(provider==="querit"){try{if(isQueritAvailable()){let queritResult=await extractWithQuerit(url,signal,options);if(queritResult)return withDeclaredLinks(queritResult)}}catch(err){if(isAbortError3(err))return abortedResult(url);if(queritError=errorMessage5(err),isConfigParseError(err))return parseErrorResult(queritError)}continue}if(provider==="kagi"){try{if(isKagiExtractAvailable()){let ssrf=loadSsrfConfig(),kagiResult=await extractWithKagi(url,signal,{timeoutMs:options?.timeoutMs,...options?.lookup?{lookup:options.lookup}:{},ssrf});if(kagiResult)return withDeclaredLinks(kagiResult)}}catch(err){if(isAbortError3(err))return abortedResult(url);if(kagiError=errorMessage5(err),isConfigParseError(err))return parseErrorResult(kagiError)}continue}if(provider==="ollama"){try{if(isOllamaFetchAvailable()){let ssrf=loadSsrfConfig(),ollamaResult=await extractWithOllama(url,signal,{timeoutMs:options?.timeoutMs,...options?.lookup?{lookup:options.lookup}:{},ssrf});if(ollamaResult)return withDeclaredLinks(ollamaResult)}}catch(err){if(isAbortError3(err))return abortedResult(url);if(ollamaError=errorMessage5(err),isConfigParseError(err))return parseErrorResult(ollamaError)}continue}if(provider==="parallel"){try{if(isParallelAvailable()){let parallelResult=await extractWithParallel(url,signal,options);if(parallelResult)return withDeclaredLinks(parallelResult)}}catch(err){if(isAbortError3(err))return abortedResult(url);if(parallelError=errorMessage5(err),isConfigParseError(err))return parseErrorResult(parallelError)}continue}if(provider==="parallel-mcp"){try{let parallelMcpResult=await extractWithParallelMcp(url,signal,options);if(parallelMcpResult)return withDeclaredLinks(parallelMcpResult)}catch(err){if(isAbortError3(err))return abortedResult(url);if(parallelMcpError=errorMessage5(err),isConfigParseError(err))return parseErrorResult(parallelMcpError)}continue}if(provider==="brightdata"){try{if(isBrightDataUnlockerAvailable()){let ssrf=loadSsrfConfig(),brightdataResult=await extractWithBrightDataUnlocker(url,signal,{timeoutMs:options?.timeoutMs,...options?.lookup?{lookup:options.lookup}:{},ssrf});if(brightdataResult)return withDeclaredLinks(brightdataResult)}}catch(err){if(isAbortError3(err))return abortedResult(url);if(brightdataError=errorMessage5(err),isConfigParseError(err))return parseErrorResult(brightdataError)}continue}if(provider==="gemini"){let geminiResult=null;try{geminiResult=await extractWithUrlContext(url,signal)??await extractWithGeminiWeb(url,signal)}catch(err){if(isAbortError3(err))return abortedResult(url);if(err instanceof CredentialResolutionError||isConfigParseError(err))return parseErrorResult(errorMessage5(err))}if(geminiResult)return withDeclaredLinks(geminiResult)}}if(signal?.aborted)return abortedResult(url);let finalHttpResult=httpResult;if(finalHttpResult&&declaredLinks.length>0)return{...finalHttpResult,error:null};if(finalHttpResult?.status===404||finalHttpResult?.status===410)return{...finalHttpResult,error:notFoundGuidance(finalHttpResult,options?.toolNames)};let searchToolName=options?.toolNames?.webSearch,guidance=[finalHttpResult?.error??"No fetch_content provider returned content",...firecrawlError?[`Firecrawl fallback failed: ${firecrawlError}`]:[],...crawl4aiError?[`Crawl4AI fallback failed: ${crawl4aiError}`]:[],...tinyfishError?[`TinyFish fallback failed: ${tinyfishError}`]:[],...search1apiError?[`Search1API fallback failed: ${search1apiError}`]:[],...queritError?[`Querit fallback failed: ${queritError}`]:[],...kagiError?[`Kagi fallback failed: ${kagiError}`]:[],...ollamaError?[`Ollama fallback failed: ${ollamaError}`]:[],...parallelError?[`Parallel fallback failed: ${parallelError}`]:[],...parallelMcpError?[`Parallel MCP fallback failed: ${parallelMcpError}`]:[],...brightdataError?[`Bright Data fallback failed: ${brightdataError}`]:[],"","Fallback options:",`  \u2022 Set firecrawlBaseUrl in ${WEB_SEARCH_CONFIG_PATH} to a self-hosted Firecrawl instance`,`  \u2022 Set crawl4aiBaseUrl in ${WEB_SEARCH_CONFIG_PATH} to a self-hosted Crawl4AI instance`,`  \u2022 Set tinyfishApiKey in ${WEB_SEARCH_CONFIG_PATH} or TINYFISH_API_KEY`,`  \u2022 Set search1apiApiKey in ${WEB_SEARCH_CONFIG_PATH} or SEARCH1API_KEY`,`  \u2022 Set queritApiKey in ${WEB_SEARCH_CONFIG_PATH} or QUERIT_API_KEY`,`  \u2022 Set kagiApiKey in ${WEB_SEARCH_CONFIG_PATH} or KAGI_API_KEY`,`  \u2022 Set ollamaApiKey in ${WEB_SEARCH_CONFIG_PATH} or OLLAMA_API_KEY`,`  \u2022 Set parallelApiKey in ${WEB_SEARCH_CONFIG_PATH} or PARALLEL_API_KEY`,`  \u2022 Set brightdataApiKey and brightdataUnlockerZone in ${WEB_SEARCH_CONFIG_PATH} or BRIGHTDATA_API_KEY and BRIGHTDATA_UNLOCKER_ZONE`,`  \u2022 Set GEMINI_API_KEY in ${WEB_SEARCH_CONFIG_PATH}`,"  \u2022 Sign into gemini.google.com in Chrome",...searchToolName?[`  \u2022 Use ${searchToolName} to find content about this topic`]:[]].join(`
+`);return{...finalHttpResult??{url,title:"",content:"",error:null},error:guidance}}function isLikelyJSRendered(html){let bodyMatch=html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);if(!bodyMatch)return!1;let textContent=bodyMatch[1].replace(/<script[\s\S]*?<\/script>/gi,"").replace(/<style[\s\S]*?<\/style>/gi,"").replace(/<[^>]+>/g,"").replace(/\s+/g," ").trim(),scriptCount=(html.match(/<script/gi)||[]).length;return textContent.length<500&&scriptCount>3}async function readPDFResponseBuffer(response,maxSizeMB){let maxBytes=maxSizeMB*1024*1024;return readResponseBufferWithLimit(response,maxBytes,()=>pdfSizeLimitError(maxSizeMB))}async function readTextResponseWithLimit(response,maxBytes){let buffer=await readResponseBufferWithLimit(response,maxBytes,()=>responseSizeLimitError(maxBytes)),charset=response.headers.get("content-type")?.match(/charset\s*=\s*["']?([^;"'\s]+)/i)?.[1];try{return new TextDecoder(charset||"utf-8").decode(buffer)}catch{return new TextDecoder("utf-8").decode(buffer)}}function isTextContentType(contentType){let mimeType=contentType.split(";",1)[0]?.trim().toLowerCase()??"";return mimeType.startsWith("text/")||mimeType==="application/json"||mimeType==="application/ld+json"||mimeType==="application/xml"||mimeType==="application/xhtml+xml"||mimeType==="application/javascript"||mimeType==="application/x-javascript"||mimeType.endsWith("+json")||mimeType.endsWith("+xml")}async function readResponseBufferWithLimit(response,maxBytes,buildError){let reader=response.body?.getReader();if(!reader){let buffer=await response.arrayBuffer();if(buffer.byteLength>maxBytes)throw buildError();return buffer}let chunks=[],totalBytes=0;try{for(;;){let{done,value}=await reader.read();if(done)break;if(value){if(totalBytes+=value.byteLength,totalBytes>maxBytes)throw await reader.cancel(),buildError();chunks.push(value)}}}finally{reader.releaseLock()}let combined=new Uint8Array(totalBytes),offset=0;for(let chunk of chunks)combined.set(chunk,offset),offset+=chunk.byteLength;return combined.buffer}function pdfSizeLimitError(maxSizeMB){return new Error(`PDF exceeds configured pdf.maxSizeMB limit (${maxSizeMB} MB)`)}function responseSizeLimitError(maxBytes){return new Error(`Response too large (${Math.round(maxBytes/1024/1024)}MB)`)}async function extractViaHttp(url,timeoutMs,signal,options){let activityId=activityMonitor.logStart({type:"fetch",url}),controller=new AbortController,startedAt=Date.now(),timeoutId=setTimeout(()=>controller.abort(),timeoutMs),onAbort=()=>controller.abort();signal?.addEventListener("abort",onAbort);try{let ssrf=loadSsrfConfig(),domainPolicy=loadFetchContentDomainPolicy(),authProfile=options?.authFetchProfile,trustEnvProxy=options?.proxy===void 0&&ssrf.trustEnvProxy,requestInit={signal:controller.signal,__proxy:options?.proxy,headers:{"User-Agent":"OpenAI File Downloader, XaiImageApiFetch/1.0",Accept:"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8","Accept-Language":"en-US,en;q=0.9","Cache-Control":"no-cache","Sec-Fetch-Dest":"document","Sec-Fetch-Mode":"navigate","Sec-Fetch-Site":"none","Sec-Fetch-User":"?1","Upgrade-Insecure-Requests":"1"}},response=authProfile?await fetchAuthenticatedRemoteUrl(url,requestInit,{ssrf:{...ssrf,trustEnvProxy},domainPolicy,...options?.lookup?{lookup:options.lookup}:{}},authProfile):await fetchRemoteUrl(url,requestInit,{allowRanges:ssrf.allowRanges,trustEnvProxy,domainPolicy,...options?.lookup?{lookup:options.lookup}:{}});if(!response.ok&&options?.mode!=="raw")return activityMonitor.logComplete(activityId,response.status),{url,title:"",content:"",error:`HTTP ${response.status}: ${response.statusText}`,status:response.status};let contentLengthHeader=response.headers.get("content-length"),contentType=response.headers.get("content-type")||"",mimeType=contentType.split(";",1)[0]?.trim().toLowerCase()??"",isPDFContent=isPDF(url,contentType),pdfConfig=isPDFContent?loadPDFConfig():null;if(isPDFContent&&pdfConfig&&!pdfConfig.enabled)return activityMonitor.logComplete(activityId,response.status),{url,title:"",content:"",error:"PDF extraction is disabled by pdf.enabled",mimeType,status:response.status};let maxResponseSize=(pdfConfig?.maxSizeMB??5)*1024*1024;if(contentLengthHeader){let contentLength=Number.parseInt(contentLengthHeader,10);if(Number.isFinite(contentLength)&&contentLength>maxResponseSize)return activityMonitor.logComplete(activityId,response.status),{url,title:"",content:"",error:pdfConfig?pdfSizeLimitError(pdfConfig.maxSizeMB).message:`Response too large (${Math.round(contentLength/1024/1024)}MB)`}}if(options?.mode==="raw"){if(!isTextContentType(contentType))return activityMonitor.logComplete(activityId,response.status),{url,title:"",content:"",error:`Unsupported content type in raw mode: ${mimeType||"missing"}`,mimeType,status:response.status};let text2=await readTextResponseWithLimit(response,maxResponseSize);return activityMonitor.logComplete(activityId,response.status),{url,title:extractTextTitle(text2,url),content:text2,error:null,mimeType,status:response.status}}if(SUPPORTED_IMAGE_TYPES.has(mimeType)){let disabled=imageGateError();if(disabled)return activityMonitor.logComplete(activityId,response.status),{url,title:"",content:"",error:disabled,mimeType,status:response.status};try{let buffer=await readResponseBufferWithLimit(response,maxResponseSize,()=>responseSizeLimitError(maxResponseSize)),{resizeImage}=await import("./host-WP3MVKS2.js"),resized=await resizeImage(new Uint8Array(buffer),mimeType,{maxWidth:2e3,maxHeight:2e3});if(activityMonitor.logComplete(activityId,response.status),!resized)return{url,title:"",content:"",error:`Could not decode image: ${mimeType}`,mimeType,status:response.status};let title=new URL(response.url||url).pathname.split("/").pop()||url;return{url,title,content:`Image fetched (${resized.width}\xD7${resized.height}, ${resized.mimeType})`,error:null,thumbnail:{data:resized.data,mimeType:resized.mimeType},mimeType:resized.mimeType,status:response.status}}catch(err){let message=errorMessage5(err);return activityMonitor.logError(activityId,message),{url,title:"",content:"",error:message,mimeType,status:response.status}}}if(isPDFContent&&pdfConfig)try{let buffer=await readPDFResponseBuffer(response,pdfConfig.maxSizeMB);if(signal?.aborted)return abortedResult(url);let result=await extractPDFToMarkdown(buffer,url,{signal});return activityMonitor.logComplete(activityId,response.status),{url,title:result.title,content:`PDF extracted and saved to: ${result.outputPath}
 
 Pages: ${result.pages}
-Characters: ${result.chars}`,error:null}}catch(err){let message=err instanceof Error?err.message:String(err);return activityMonitor.logError(activityId,message),message.startsWith("PDF exceeds configured pdf.maxSizeMB limit")?{url,title:"",content:"",error:message}:err instanceof CredentialResolutionError||isConfigParseError(err)?{url,title:"",content:"",error:message}:{url,title:"",content:"",error:`PDF extraction failed: ${message}`}}if(contentType.includes("application/octet-stream")||contentType.includes("image/")||contentType.includes("audio/")||contentType.includes("video/")||contentType.includes("application/zip"))return activityMonitor.logComplete(activityId,response.status),{url,title:"",content:"",error:`Unsupported content type: ${contentType.split(";")[0]}`};let text=await readTextResponseWithLimit(response,maxResponseSize);if(!(contentType.includes("text/html")||contentType.includes("application/xhtml+xml"))){activityMonitor.logComplete(activityId,response.status);let title=extractTextTitle(text,url);return{url,title,content:text,error:null}}let{document}=parseHTML(text),documentTitle=document.title?.trim()??"",declaredLinks=discoverDeclaredWebLinks(document,response.headers.get("link"),response.url||url),article=new import_readability.Readability(document).parse();if(!article){let rscResult=extractRSCContent(text);if(rscResult&&rscResult.content.length>=MIN_USEFUL_CONTENT)return activityMonitor.logComplete(activityId,response.status),{url,title:rscResult.title,content:appendDeclaredWebLinks(rscResult.content,declaredLinks),error:null,declaredLinks};controller.signal.throwIfAborted();let defuddleResult=await extractWithDefuddle(text,response.url||url);if(controller.signal.throwIfAborted(),defuddleResult&&defuddleResult.content.length>=MIN_USEFUL_CONTENT)return activityMonitor.logComplete(activityId,response.status),{url,title:documentTitle||defuddleResult.title,content:appendDeclaredWebLinks(defuddleResult.content,declaredLinks),error:null,declaredLinks};activityMonitor.logComplete(activityId,response.status);let errorMsg=isLikelyJSRendered(text)?"Page appears to be JavaScript-rendered (content loads dynamically)":"Could not extract readable content from HTML structure";return{url,title:documentTitle,content:appendDeclaredWebLinks("",declaredLinks),error:errorMsg,declaredLinks}}if(typeof article.content!="string")throw new Error("Readability returned invalid article content");let markdown=turndown.turndown(article.content);if(activityMonitor.logComplete(activityId,response.status),markdown.length<MIN_USEFUL_CONTENT){let rscResult=extractRSCContent(text);if(rscResult&&rscResult.content.length>=MIN_USEFUL_CONTENT)return{url,title:rscResult.title,content:appendDeclaredWebLinks(rscResult.content,declaredLinks),error:null,declaredLinks};controller.signal.throwIfAborted();let defuddleResult=await extractWithDefuddle(text,response.url||url);return controller.signal.throwIfAborted(),defuddleResult&&defuddleResult.content.length>=MIN_USEFUL_CONTENT?{url,title:article.title||documentTitle||defuddleResult.title,content:appendDeclaredWebLinks(defuddleResult.content,declaredLinks),error:null,declaredLinks}:{url,title:article.title||documentTitle,content:appendDeclaredWebLinks(markdown,declaredLinks),error:isLikelyJSRendered(text)?"Page appears to be JavaScript-rendered (content loads dynamically)":"Extracted content appears incomplete",declaredLinks}}return{url,title:article.title||documentTitle,content:appendDeclaredWebLinks(markdown,declaredLinks),error:null,declaredLinks}}catch(err){let message=err instanceof Error?err.message:String(err);return message.toLowerCase().includes("abort")?activityMonitor.logComplete(activityId,0):activityMonitor.logError(activityId,message),{url,title:"",content:"",error:message}}finally{clearTimeout(timeoutId),signal?.removeEventListener("abort",onAbort)}}function extractHeadingTitle(text){let match=text.match(/^#{1,2}\s+(.+)/m);return match&&match[1].replace(/\*+/g,"").trim()||null}function extractTextTitle(text,url){return extractHeadingTitle(text)??(new URL(url).pathname.split("/").pop()||url)}async function fetchAllContent(urls,signal,options){let results=await Promise.all(urls.map(url=>fetchLimit(()=>extractContent(url,signal,options))));return options?.mode==="raw"?results:results.map((result,index)=>{if(!result.content)return result;let sanitized=sanitizeInlineDataUris(result.content,`urls[${index}].content`);return sanitized.omissions.length>0?{...result,content:sanitized.text}:result})}export{extractContent,extractHeadingTitle,fetchAllContent,loadSsrfAllowRanges,loadSsrfConfig,readPDFResponseBuffer,resolveFetchTimeoutMs};
+Characters: ${result.chars}`,error:null}}catch(err){let message=err instanceof Error?err.message:String(err);return activityMonitor.logError(activityId,message),message.startsWith("PDF exceeds configured pdf.maxSizeMB limit")?{url,title:"",content:"",error:message}:err instanceof CredentialResolutionError||isConfigParseError(err)?{url,title:"",content:"",error:message}:{url,title:"",content:"",error:`PDF extraction failed: ${message}`}}if(contentType.includes("application/octet-stream")||contentType.includes("image/")||contentType.includes("audio/")||contentType.includes("video/")||contentType.includes("application/zip"))return activityMonitor.logComplete(activityId,response.status),{url,title:"",content:"",error:`Unsupported content type: ${contentType.split(";")[0]}`};let text=await readTextResponseWithLimit(response,maxResponseSize);if(!(contentType.includes("text/html")||contentType.includes("application/xhtml+xml"))){activityMonitor.logComplete(activityId,response.status);let title=extractTextTitle(text,url);return{url,title,content:text,error:null}}let{parseHTML}=await import("./esm-E7UVEYTO.js"),{document}=parseHTML(text),documentTitle=document.title?.trim()??"",declaredLinks=discoverDeclaredWebLinks(document,response.headers.get("link"),response.url||url),{Readability}=await import("./readability-LAGF34IB.js"),article=new Readability(document).parse();if(!article){let rscResult=extractRSCContent(text);if(rscResult&&rscResult.content.length>=MIN_USEFUL_CONTENT)return activityMonitor.logComplete(activityId,response.status),{url,title:rscResult.title,content:appendDeclaredWebLinks(rscResult.content,declaredLinks),error:null,declaredLinks};controller.signal.throwIfAborted();let defuddleResult=await extractWithDefuddle(text,response.url||url);if(controller.signal.throwIfAborted(),defuddleResult&&defuddleResult.content.length>=MIN_USEFUL_CONTENT)return activityMonitor.logComplete(activityId,response.status),{url,title:documentTitle||defuddleResult.title,content:appendDeclaredWebLinks(defuddleResult.content,declaredLinks),error:null,declaredLinks};activityMonitor.logComplete(activityId,response.status);let errorMsg=isLikelyJSRendered(text)?"Page appears to be JavaScript-rendered (content loads dynamically)":"Could not extract readable content from HTML structure";return{url,title:documentTitle,content:appendDeclaredWebLinks("",declaredLinks),error:errorMsg,declaredLinks}}if(typeof article.content!="string")throw new Error("Readability returned invalid article content");let markdown=(await getTurndown()).turndown(article.content);if(activityMonitor.logComplete(activityId,response.status),markdown.length<MIN_USEFUL_CONTENT){let rscResult=extractRSCContent(text);if(rscResult&&rscResult.content.length>=MIN_USEFUL_CONTENT)return{url,title:rscResult.title,content:appendDeclaredWebLinks(rscResult.content,declaredLinks),error:null,declaredLinks};controller.signal.throwIfAborted();let defuddleResult=await extractWithDefuddle(text,response.url||url);return controller.signal.throwIfAborted(),defuddleResult&&defuddleResult.content.length>=MIN_USEFUL_CONTENT?{url,title:article.title||documentTitle||defuddleResult.title,content:appendDeclaredWebLinks(defuddleResult.content,declaredLinks),error:null,declaredLinks}:{url,title:article.title||documentTitle,content:appendDeclaredWebLinks(markdown,declaredLinks),error:isLikelyJSRendered(text)?"Page appears to be JavaScript-rendered (content loads dynamically)":"Extracted content appears incomplete",declaredLinks}}return{url,title:article.title||documentTitle,content:appendDeclaredWebLinks(markdown,declaredLinks),error:null,declaredLinks}}catch(err){let message=err instanceof Error?err.message:String(err);return message.toLowerCase().includes("abort")?activityMonitor.logComplete(activityId,0):activityMonitor.logError(activityId,message),{url,title:"",content:"",error:message}}finally{if(clearTimeout(timeoutId),signal?.removeEventListener("abort",onAbort),signal?.aborted)return abortedResult(url);if(controller.signal.aborted||Date.now()-startedAt>=timeoutMs)return{url,title:"",content:"",error:"The operation was aborted."}}}function extractHeadingTitle(text){let match=text.match(/^#{1,2}\s+(.+)/m);return match&&match[1].replace(/\*+/g,"").trim()||null}function extractTextTitle(text,url){return extractHeadingTitle(text)??(new URL(url).pathname.split("/").pop()||url)}async function fetchAllContent(urls,signal,options){let results=await Promise.all(urls.map(url=>fetchLimit(()=>extractContent(url,signal,options))));return options?.mode==="raw"?results:results.map((result,index)=>{if(!result.content)return result;let sanitized=sanitizeInlineDataUris(result.content,`urls[${index}].content`);return sanitized.omissions.length>0?{...result,content:sanitized.text}:result})}export{extractContent,extractHeadingTitle,fetchAllContent,loadSsrfAllowRanges,loadSsrfConfig,readPDFResponseBuffer,resolveFetchTimeoutMs};