@duckmind/dm-darwin-arm64 0.52.3 → 0.52.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dm +0 -0
- package/extensions/.dm-extensions.json +6 -3
- package/extensions/greedysearch-dm/bin/gschrome.mjs +5 -7
- package/extensions/greedysearch-dm/bin/launch-visible.mjs +2 -4
- package/extensions/greedysearch-dm/bin/launch.mjs +11 -5
- package/extensions/greedysearch-dm/bin/search.mjs +122 -120
- package/extensions/greedysearch-dm/extractors/consensus.mjs +43 -43
- package/extensions/greedysearch-dm/extractors/gemini.mjs +44 -44
- package/extensions/greedysearch-dm/extractors/logically.mjs +41 -41
- package/extensions/greedysearch-dm/index.js +18 -18
- package/extensions/greedysearch-dm/src/search/browser-lifecycle.mjs +8 -9
- package/extensions/greedysearch-dm/src/search/chrome.mjs +25 -25
- package/extensions/greedysearch-dm/src/search/constants.mjs +8 -8
- package/extensions/greedysearch-dm/src/search/engines.mjs +9 -9
- package/extensions/greedysearch-dm/src/search/fetch-source.mjs +49 -49
- package/extensions/greedysearch-dm/src/search/launcher-paths.mjs +3 -0
- package/extensions/greedysearch-dm/src/search/partial-output.mjs +1 -0
- package/extensions/greedysearch-dm/src/search/port-pid.mjs +1 -0
- package/extensions/greedysearch-dm/src/search/recovery.mjs +1 -1
- package/extensions/greedysearch-dm/src/search/research.mjs +101 -101
- package/extensions/greedysearch-dm/src/search/synthesis-runner.mjs +11 -11
- package/extensions/greedysearch-dm/src/search/synthesis.mjs +10 -10
- package/extensions/greedysearch-dm/src/tools/greedy-search-handler.js +18 -18
- package/extensions/greedysearch-dm/src/tools/shared.js +9 -9
- package/package.json +1 -1
|
@@ -1,18 +1,18 @@
|
|
|
1
|
-
import{createRequire as
|
|
1
|
+
import{createRequire as Vr}from"node:module";var Jr=Object.defineProperty;var Yr=(t)=>t;function dr(t,e){this[t]=Yr.bind(null,e)}var et=(t,e)=>{for(var r in e)Jr(t,r,{get:e[r],enumerable:!0,configurable:!0,set:dr.bind(e,r)})};var ot=(t,e)=>()=>(t&&(e=t(t=0)),e);var Zr=Vr(import.meta.url);var fr={};et(fr,{fetchTopSource:()=>Da,fetchSourceContent:()=>ir,fetchMultipleSources:()=>ya});import{createRequire as bn}from"node:module";import{Readability as sn}from"@mozilla/readability";import{JSDOM as _n}from"jsdom";import En from"turndown";import{platform as Ei,tmpdir as Fn}from"node:os";import{spawn as Gn}from"node:child_process";import{dirname as In,join as Jn}from"node:path";import{fileURLToPath as Yn}from"node:url";import{basename as dn}from"node:path";import{existsSync as mt,mkdirSync as Vn,readFileSync as Ue,writeFileSync as Zn}from"node:fs";import{homedir as Kn}from"node:os";import{join as Xe}from"node:path";import{tmpdir as _t}from"node:os";import{platform as Ri,tmpdir as Bn}from"node:os";import{existsSync as bt,mkdirSync as Un,readFileSync as xe,writeFileSync as Xn}from"node:fs";import{homedir as xn}from"node:os";import{join as Qe}from"node:path";import{tmpdir as Et}from"node:os";import{existsSync as st,mkdirSync as fa,readFileSync as er,writeFileSync as oa}from"node:fs";import{homedir as ga}from"node:os";import{join as rr}from"node:path";import{tmpdir as yt}from"node:os";function Dn(t={}){return{...Je,...t}}function Ye(t){try{let e=new URL(t),r=e.hostname.toLowerCase();for(let n of yn)if(n.test(r))return{blocked:!0,reason:`Private/internal address: ${r}`};if(e.protocol==="file:")return{blocked:!0,reason:"File protocol not allowed"};return{blocked:!1}}catch(e){return{blocked:!0,reason:`Invalid URL: ${e.message}`}}}function vn(t){try{let e=new URL(t);if(!(e.hostname==="github.com"||e.hostname.endsWith(".github.com")))return t;let r=e.pathname.split("/").filter(Boolean);if(r.length<5)return t;let[n,a,l,i,...f]=r;if(l!=="blob")return t;let g=f.join("/");return`https://raw.githubusercontent.com/${n}/${a}/${i}/${g}`}catch{return t}}async function kn(t,e={}){let r=Ye(t);if(r.blocked)return{ok:!1,url:t,finalUrl:t,status:403,error:`Blocked: ${r.reason}`,needsBrowser:!1};let n=t;if(t=vn(t),t!==n)console.error(`[fetcher] Rewrote GitHub URL: ${n.slice(0,60)}... → raw.githubusercontent.com`);let{timeoutMs:a=15000,userAgent:l,signal:i}=e,f=new AbortController,g=setTimeout(()=>f.abort(),a);if(i)i.addEventListener("abort",()=>f.abort(),{once:!0});try{let o=await fetch(t,{method:"GET",headers:{...Je,"user-agent":l||Ie},redirect:"follow",signal:f.signal});clearTimeout(g);let p=o.headers.get("content-type")||"",h=o.url,m=o.headers.get("last-modified")||"",w=!1;try{w=new URL(h).hostname.toLowerCase()==="raw.githubusercontent.com"}catch{}if(p.includes("text/plain")&&w){let C=await o.text();return{ok:!0,url:n,finalUrl:h,status:o.status,title:h.split("/").pop()||"GitHub File",byline:"",siteName:"GitHub",lang:"",publishedTime:m,lastModified:m,markdown:C,contentLength:C.length,excerpt:C.slice(0,300).replaceAll(/\n/g," "),needsBrowser:!1}}if(!p.includes("text/html")&&!p.includes("application/xhtml"))return{ok:!1,url:t,finalUrl:h,status:o.status,error:`Unsupported content type: ${p}`,needsBrowser:!1};let s=await o.text(),b=de(o.status,s,h,t);if(b.blocked)return{ok:!1,url:t,finalUrl:h,status:o.status,error:`Blocked: ${b.reason}`,needsBrowser:!0};let c=Ve(s,h),E=Ze(c);if(!E.ok)return{ok:!1,url:t,finalUrl:h,status:o.status,error:`Low quality content: ${E.reason}`,needsBrowser:!0};return{ok:!0,url:t,finalUrl:h,status:o.status,title:c.title,byline:c.byline,siteName:c.siteName,lang:c.lang,publishedTime:c.publishedTime||m,lastModified:m,markdown:c.markdown,excerpt:c.excerpt,contentLength:c.markdown.length,needsBrowser:!1}}catch(o){clearTimeout(g);let p=Pn(o);return{ok:!1,url:t,finalUrl:t,status:0,error:o.message,needsBrowser:p}}}function de(t,e,r,n){let a=e.match(/<title[^>]*>([^<]*)<\/title>/i)?.[1]?.toLowerCase()||"",l=e.slice(0,30000).toLowerCase(),i=`${a} ${l}`;if(t===403||t===429||t===503)return{blocked:!0,reason:`HTTP ${t}`};let f=[{pattern:/class=["'][^"']*captcha["']|<div[^>]*id=["']captcha/i,reason:"captcha"},{pattern:/g-recaptcha|data-sitekey|i['"]m not a robot/i,reason:"captcha"},{pattern:/checking your browser.{0,100}please wait|cf-browser-verification/i,reason:"cloudflare challenge"},{pattern:/just a moment.{0,50}security check|ddos protection by cloudflare/i,reason:"cloudflare challenge"},{pattern:/unusual traffic.{0,50}from your computer network/i,reason:"unusual traffic"},{pattern:/bot detected|automated.{0,20}request/i,reason:"bot detection"},{pattern:/enable\s+javascript\s+to\s+view|javascript\s+is\s+required.{0,50}enabled/i,reason:"requires javascript"},{pattern:/access denied|accessdenied/i,reason:"access denied"},{pattern:/protected by anubis|anubis uses a proof-of-work/i,reason:"anubis challenge"}];for(let o of f)if(o.pattern.test(i))return{blocked:!0,reason:o.reason};let g=On(n,r,e);if(g)return{blocked:!0,reason:g};return{blocked:!1}}function On(t,e,r){try{let n=new URL(t),a=new URL(e);if(n.hostname.toLowerCase()===a.hostname.toLowerCase())return;let l=a.hostname.toLowerCase();if(jn.some((f)=>l===f||l.endsWith(`.${f}`)))return`redirected to login (${a.hostname})`;if(Cn.some((f)=>l.startsWith(f)))return`redirected to login (${a.hostname})`;let i=r.slice(0,20000).toLowerCase();if(Nn.some((f)=>i.includes(f)))return`redirected to login page (${a.hostname})`}catch{}return}function Pn(t){let e=t.message.toLowerCase();return e.includes("fetch failed")||e.includes("unable to verify")||e.includes("certificate")||e.includes("timeout")}function Re(t){let e=['meta[property="article:published_time"]','meta[name="article:published_time"]','meta[property="og:published_time"]','meta[name="publication_date"]','meta[name="date"]','meta[itemprop="datePublished"]','time[itemprop="datePublished"]','meta[name="DC.date"]'];for(let r of e){let n=t.querySelector(r),a=n?.getAttribute("content")||n?.getAttribute("datetime")||"";if(a)return a}return""}function Ve(t,e){let r=new _n(t,{url:e}).window.document,n=new sn(r).parse();if(n&&n.content){let l=Ge.turndown(n.content).replaceAll(/\n{3,}/g,`
|
|
2
2
|
|
|
3
|
-
`).trim(),i=n.publishedTime||
|
|
3
|
+
`).trim(),i=n.publishedTime||Re(r)||"";return{title:n.title||r.title||e,byline:n.byline||"",siteName:n.siteName||"",lang:n.lang||"",publishedTime:i,markdown:l,excerpt:l.slice(0,300).replaceAll(/\n/g," ")}}let a=r.body;if(a){let l=a.cloneNode(!0);l.querySelectorAll("script, style, nav, footer, header, aside").forEach((f)=>f.remove());let i=(l.textContent||"").replaceAll(/\s+/g," ").trim();return{title:r.title||e,byline:"",siteName:"",lang:"",publishedTime:Re(r),markdown:i,excerpt:i.slice(0,300)}}return{title:e,byline:"",siteName:"",lang:"",publishedTime:"",markdown:"",excerpt:""}}function Ze(t){let e=t.markdown.trim().toLowerCase(),r=(t.title||"").toLowerCase();if(t.markdown.trim().length<100)return{ok:!1,reason:"content too short (< 100 chars)"};let n=e.toLowerCase(),a=[{check:()=>n.includes("loading")&&n.includes("please wait"),desc:"loading page"},{check:()=>n.includes("please ensure javascript is enabled"),desc:"requires javascript"},{check:()=>n.includes("enable javascript to view"),desc:"requires javascript"},{check:()=>n.includes("just a moment"),desc:"cloudflare challenge detected in content"},{check:()=>n.includes("verify you are human"),desc:"human verification"},{check:()=>n.includes("captcha required"),desc:"captcha in extracted content"},{check:()=>n.includes("access denied"),desc:"access denied in content"},{check:()=>/^\s{0,10}sign\s{1,5}in\s{0,10}$|^\s{0,10}log\s{1,5}in\s{0,10}$/im.test(e),desc:"login form only"}];for(let{check:l,desc:i}of a)if(l())return{ok:!1,reason:i};if(r.includes("just a moment")||r.includes("checking your browser"))return{ok:!1,reason:"cloudflare challenge page detected in title"};return{ok:!0}}function Yt(t){try{let e=new URL(t);if(!(e.hostname==="github.com"||e.hostname.endsWith(".github.com")))return null;let r=e.pathname.split("/").filter(Boolean);if(r.length<2)return null;let[n,a]=r;if(r.length===2)return{owner:n,repo:a,type:"root"};if(r.length>=4&&(r[2]==="blob"||r[2]==="tree")){let l=r[2],i=r[3],f=r.slice(4).join("/");return{owner:n,repo:a,type:l,ref:i,path:f}}return null}catch{return null}}async function K(t,e=1e4){let r=new AbortController,n=setTimeout(()=>r.abort(),e);try{let a=await fetch(`https://api.github.com${t}`,{headers:Ke,signal:r.signal});if(clearTimeout(n),!a.ok)throw Error(`GitHub API ${a.status}: ${t}`);return await a.json()}catch(a){throw clearTimeout(n),a}}async function Rn(t,e){try{let r=await K(`/repos/${t}/${e}/readme`);if(r.content&&r.encoding==="base64")return Buffer.from(r.content,"base64").toString("utf8");return""}catch{return""}}async function Te(t,e,r="HEAD",n=""){try{let a=await K(`/repos/${t}/${e}/git/ref/heads/${r==="HEAD"?"main":r}`).catch(()=>K(`/repos/${t}/${e}/git/ref/heads/master`).catch(()=>null)),l;if(a?.object?.sha)l=(await K(`/repos/${t}/${e}/git/commits/${a.object.sha}`)).tree.sha;else{let f=await K(`/repos/${t}/${e}`);l=(await K(`/repos/${t}/${e}/branches/${f.default_branch}`)).commit.commit.tree.sha}let i=(await K(`/repos/${t}/${e}/git/trees/${l}`)).tree||[];if(n)i=i.filter((f)=>f.path.startsWith(n));return i.slice(0,50).map((f)=>({path:f.path,type:f.type==="tree"?"dir":"file",size:f.size}))}catch{return[]}}async function Tn(t,e,r,n,a=1e4){let l=[`https://raw.githubusercontent.com/${t}/${e}/${r&&r!=="HEAD"?r:"main"}/${n}`,`https://raw.githubusercontent.com/${t}/${e}/master/${n}`];for(let i of l){let f=new AbortController,g=setTimeout(()=>f.abort(),a);try{let o=await fetch(i,{headers:{"user-agent":Ke["user-agent"]},signal:f.signal});if(clearTimeout(g),o.ok)return await o.text()}catch{clearTimeout(g)}}return null}async function Ln(t){let e=Yt(t);if(!e)return{ok:!1,error:"Not a valid GitHub URL"};let{owner:r,repo:n,type:a,ref:l,path:i}=e;try{if(a==="root"||a==="tree"&&!i){let[f,g,o]=await Promise.allSettled([K(`/repos/${r}/${n}`),Rn(r,n),Te(r,n,l||"HEAD")]);if(f.status==="rejected")return{ok:!1,error:f.reason?.message||"Repo not found"};let p=f.value,h=g.status==="fulfilled"?g.value:"",m=o.status==="fulfilled"?o.value:[],w=p?.description?`
|
|
4
4
|
|
|
5
|
-
> ${p.description}`:"",
|
|
5
|
+
> ${p.description}`:"",s=p?.stargazers_count==null?"":` ⭐ ${p.stargazers_count}`,b=p?.language?` · ${p.language}`:"",c=`# ${r}/${n}${s}${b}${w}
|
|
6
6
|
|
|
7
|
-
`;if(h)
|
|
7
|
+
`;if(h)c+=h.slice(0,6000);else c+=`[No README found]
|
|
8
8
|
|
|
9
9
|
Files:
|
|
10
|
-
${m.map((
|
|
11
|
-
`)}`;return{ok:!0,title:`${r}/${n}`,content:
|
|
10
|
+
${m.map((E)=>` ${E.type==="dir"?"\uD83D\uDCC1":"\uD83D\uDCC4"} ${E.path}`).join(`
|
|
11
|
+
`)}`;return{ok:!0,title:`${r}/${n}`,content:c,tree:m.slice(0,30)}}if(a==="blob"&&i){let f=await Tn(r,n,l,i);if(f===null)return{ok:!1,error:`File not found: ${i}`};return{ok:!0,title:`${r}/${n}: ${i}`,content:f}}if(a==="tree"&&i){let f=await Te(r,n,l||"HEAD",i),g=f.map((o)=>` ${o.type==="dir"?"\uD83D\uDCC1":"\uD83D\uDCC4"} ${o.path}`).join(`
|
|
12
12
|
`);return{ok:!0,title:`${r}/${n}/${i}`,content:`[Directory: ${i}]
|
|
13
13
|
|
|
14
14
|
Files:
|
|
15
|
-
${
|
|
15
|
+
${g}`,tree:f}}return{ok:!1,error:"Unsupported GitHub URL type"}}catch(f){return{ok:!1,error:f.message}}}function Mn(t){try{let e=new URL(t),r=e.hostname.toLowerCase();if(!(r==="reddit.com"||r.endsWith(".reddit.com")))return null;let n=e.pathname;if(n.match(/^\/(u|user)\/[^/]+\/?$/i))return{type:"user",cleanUrl:Le(t)};if(n.match(/^\/r\/[^/]+\/comments\/[^/]+/i))return{type:"post",cleanUrl:Le(t)};return null}catch{return null}}function Le(t){try{let e=new URL(t);return`${e.protocol}//${e.hostname}${e.pathname}`}catch{return t}}async function Wn(t,e=8000){let r=Date.now();try{let n=t.replaceAll(/\/?$/g,".json"),a=new AbortController,l=setTimeout(()=>a.abort(),15000),i=await fetch(n,{headers:An,signal:a.signal});if(clearTimeout(l),!i.ok)throw Error(`Reddit API ${i.status}`);let f=await i.json();if(!Array.isArray(f)||f.length<1)throw Error("Invalid Reddit API response structure");let g=f[0],o=f[1],p=g?.data?.children?.[0]?.data;if(!p)throw Error("No post data in Reddit response");let h=Hn(p,o,e);return{ok:!0,url:t,finalUrl:t,status:200,contentType:"text/markdown",lastModified:"",title:p.title||"Reddit Post",byline:`u/${p.author}`,siteName:`r/${p.subreddit}`,lang:"en",publishedTime:new Date(p.created_utc*1000).toISOString(),excerpt:p.selftext?.slice(0,300).replace(/\n/g," ")||"",markdown:h,contentLength:h.length,needsBrowser:!1,duration:Date.now()-r}}catch(n){return{ok:!1,url:t,finalUrl:t,status:0,error:`Reddit fetch failed: ${n.message}`,needsBrowser:!1,duration:Date.now()-r}}}function Hn(t,e,r){let n="";if(n+=`# ${t.title}
|
|
16
16
|
|
|
17
17
|
`,n+=`**Subreddit:** r/${t.subreddit} | **Author:** u/${t.author} | **Score:** ${t.score}
|
|
18
18
|
|
|
@@ -40,7 +40,7 @@ ${o}`,tree:f}}return{ok:!1,error:"Unsupported GitHub URL type"}}catch(f){return{
|
|
|
40
40
|
|
|
41
41
|
`,n=e-r.length,a=Math.floor(n*0.75),l=n-a,i=a;while(i>a-100&&t[i]!==`
|
|
42
42
|
`)i--;if(i<=a-100)i=a;let f=t.length-l;while(f<t.length-l+100&&t[f]!==`
|
|
43
|
-
`)f++;if(f>=t.length-l+100)f=t.length-l;let
|
|
43
|
+
`)f++;if(f>=t.length-l+100)f=t.length-l;let g=t.slice(0,i).trimEnd(),o=t.slice(f).trimStart();return`${g}${r}${o}`}function Qn(t=process.env,e=process.execPath){let r=t.GREEDY_SEARCH_NODE||t.NODE_BINARY||t.NODE;if(r?.trim())return r.trim();let n=dn(e||"").toLowerCase();if(n==="node"||n==="node.exe")return e;return"node"}function Se(t,e=30000){return un(t,null,e)}function un(t,e=null,r=30000){return new Promise((n,a)=>{let l=Gn(Qn(),[zn,...t],{stdio:[e==null?"ignore":"pipe","pipe","pipe"]});if(e!=null)l.stdin.write(e),l.stdin.end();let i="",f="";l.stdout.on("data",(o)=>i+=o),l.stderr.on("data",(o)=>f+=o);let g=setTimeout(()=>{l.kill(),a(Error(`cdp timeout: ${t[0]}`))},r);l.on("close",(o)=>{if(clearTimeout(g),o===0)n(i.trim());else a(Error(f.trim()||`cdp exit ${o}`))})})}async function Ae(t){await Se(["evalraw",t,"Page.addScriptToEvaluateOnNewDocument",JSON.stringify({source:`
|
|
44
44
|
(function() {
|
|
45
45
|
// ── Runtime.enable / CDP detection masking ──────────────
|
|
46
46
|
try { delete window.__REBROWSER_RUNTIME_ENABLE; } catch(_) {}
|
|
@@ -248,35 +248,35 @@ ${o}`,tree:f}}return{ok:!1,error:"Unsupported GitHub URL type"}}catch(f){return{
|
|
|
248
248
|
};
|
|
249
249
|
} catch(_) {}
|
|
250
250
|
})();
|
|
251
|
-
`})])}function
|
|
252
|
-
[greedysearch] Available engines: ${Object.keys(
|
|
253
|
-
`);if(r.length>0)return r;process.stderr.write(`[greedysearch] Warning: no valid engines in ${
|
|
254
|
-
`)}}}catch{}return Vt}function
|
|
255
|
-
`,"utf8")}catch{}}function
|
|
256
|
-
[greedysearch] Available synthesizers: ${We.join(", ")}
|
|
257
|
-
[greedysearch] Falling back to default: ${Zt}
|
|
258
|
-
`)}}}catch{}return Zt}function qn(){try{if(bt(V)){let t=Ie(V,"utf8"),e=JSON.parse(t);if(Array.isArray(e.engines)&&e.engines.length>0&&e.engines.every((r)=>typeof r==="string")){let r=e.engines.filter((a)=>Ht[a]),n=e.engines.filter((a)=>!Ht[a]);if(n.length>0)process.stderr.write(`[greedysearch] Warning: ignoring unknown engine(s) in ${V}: ${n.join(", ")}
|
|
259
|
-
[greedysearch] Available engines: ${Object.keys(Ht).join(", ")}
|
|
260
|
-
`);if(r.length>0)return r;process.stderr.write(`[greedysearch] Warning: no valid engines in ${V}, falling back to defaults: ${Kt.join(", ")}
|
|
261
|
-
`)}}}catch{}return Kt}function ta(){try{if(!bt(Bt))Bn(Bt,{recursive:!0});if(!bt(V))Kn(V,JSON.stringify({engines:Kt,synthesizer:Ut},null,2)+`
|
|
262
|
-
`,"utf8")}catch{}}function ea(){try{if(bt(V)){let t=Ie(V,"utf8"),e=JSON.parse(t);if(typeof e.synthesizer==="string"){let r=e.synthesizer.toLowerCase();if(Me.includes(r))return r;process.stderr.write(`[greedysearch] Warning: unknown synthesizer "${e.synthesizer}" in ${V}
|
|
251
|
+
`})])}function qn(t,e=9222){let r=Number.parseInt(String(t??""),10);return Number.isInteger(r)&&r>1024&&r<65535?r:e}function ta(){try{if(mt(Y)){let t=Ue(Y,"utf8"),e=JSON.parse(t);if(Array.isArray(e.engines)&&e.engines.length>0&&e.engines.every((r)=>typeof r==="string")){let r=e.engines.filter((a)=>Gt[a]),n=e.engines.filter((a)=>!Gt[a]);if(n.length>0)process.stderr.write(`[greedysearch] Warning: ignoring unknown engine(s) in ${Y}: ${n.join(", ")}
|
|
252
|
+
[greedysearch] Available engines: ${Object.keys(Gt).join(", ")}
|
|
253
|
+
`);if(r.length>0)return r;process.stderr.write(`[greedysearch] Warning: no valid engines in ${Y}, falling back to defaults: ${Vt.join(", ")}
|
|
254
|
+
`)}}}catch{}return Vt}function ea(){try{if(!mt(dt))Vn(dt,{recursive:!0});if(!mt(Y))Zn(Y,JSON.stringify({engines:Vt,synthesizer:Zt},null,2)+`
|
|
255
|
+
`,"utf8")}catch{}}function ra(){try{if(mt(Y)){let t=Ue(Y,"utf8"),e=JSON.parse(t);if(typeof e.synthesizer==="string"){let r=e.synthesizer.toLowerCase();if(Me.includes(r))return r;process.stderr.write(`[greedysearch] Warning: unknown synthesizer "${e.synthesizer}" in ${Y}
|
|
263
256
|
[greedysearch] Available synthesizers: ${Me.join(", ")}
|
|
257
|
+
[greedysearch] Falling back to default: ${Zt}
|
|
258
|
+
`)}}}catch{}return Zt}function na(t,e=9222){let r=Number.parseInt(String(t??""),10);return Number.isInteger(r)&&r>1024&&r<65535?r:e}function aa(){try{if(bt(d)){let t=xe(d,"utf8"),e=JSON.parse(t);if(Array.isArray(e.engines)&&e.engines.length>0&&e.engines.every((r)=>typeof r==="string")){let r=e.engines.filter((a)=>It[a]),n=e.engines.filter((a)=>!It[a]);if(n.length>0)process.stderr.write(`[greedysearch] Warning: ignoring unknown engine(s) in ${d}: ${n.join(", ")}
|
|
259
|
+
[greedysearch] Available engines: ${Object.keys(It).join(", ")}
|
|
260
|
+
`);if(r.length>0)return r;process.stderr.write(`[greedysearch] Warning: no valid engines in ${d}, falling back to defaults: ${Bt.join(", ")}
|
|
261
|
+
`)}}}catch{}return Bt}function la(){try{if(!bt(Kt))Un(Kt,{recursive:!0});if(!bt(d))Xn(d,JSON.stringify({engines:Bt,synthesizer:Ut},null,2)+`
|
|
262
|
+
`,"utf8")}catch{}}function ia(){try{if(bt(d)){let t=xe(d,"utf8"),e=JSON.parse(t);if(typeof e.synthesizer==="string"){let r=e.synthesizer.toLowerCase();if(We.includes(r))return r;process.stderr.write(`[greedysearch] Warning: unknown synthesizer "${e.synthesizer}" in ${d}
|
|
263
|
+
[greedysearch] Available synthesizers: ${We.join(", ")}
|
|
264
264
|
[greedysearch] Falling back to default: ${Ut}
|
|
265
|
-
`)}}}catch{}return Ut}async function
|
|
266
|
-
`)[0];if(!t)throw Error("No Chrome tabs found");return t.slice(0,8)}async function
|
|
267
|
-
[greedysearch] Available engines: ${Object.keys(
|
|
268
|
-
`);if(r.length>0)return r;process.stderr.write(`[greedysearch] Warning: no valid engines in ${
|
|
269
|
-
`)}}}catch{}return
|
|
270
|
-
`,"utf8")}catch{}}function
|
|
271
|
-
[greedysearch] Available synthesizers: ${
|
|
272
|
-
[greedysearch] Falling back to default: ${
|
|
273
|
-
`)}}}catch{}return
|
|
274
|
-
|
|
275
|
-
${a}`,pages:n.total}}catch(r){return{error:r.message||String(r)}}}function
|
|
276
|
-
`)}}if(
|
|
265
|
+
`)}}}catch{}return Ut}async function tr(){let t=(await R(["list"])).split(`
|
|
266
|
+
`)[0];if(!t)throw Error("No Chrome tabs found");return t.slice(0,8)}async function St(t="about:blank"){let e=await tr(),r=new URL(t).hostname;if(r==="copilot.microsoft.com"||r==="www.perplexity.ai"||r==="perplexity.ai"||r.endsWith(".perplexity.ai")){let l=await R(["evalraw",e,"Target.createTarget",JSON.stringify({url:"about:blank"})]),{targetId:i}=JSON.parse(l),f=i.slice(0,8);if(await R(["list"]).catch(()=>null),r==="copilot.microsoft.com")await Ae(f);else Ae(f).catch(()=>{});return await R(["list"]).catch(()=>null),i}let n=await R(["evalraw",e,"Target.createTarget",JSON.stringify({url:t})]),{targetId:a}=JSON.parse(n);return await R(["list"]).catch(()=>null),a}async function zt(t){try{let e=await tr();await R(["evalraw",e,"Target.closeTarget",JSON.stringify({targetId:t})])}catch{}}function pa(t,e=9222){let r=Number.parseInt(String(t??""),10);return Number.isInteger(r)&&r>1024&&r<65535?r:e}function ha(){try{if(st(V)){let t=er(V,"utf8"),e=JSON.parse(t);if(Array.isArray(e.engines)&&e.engines.length>0&&e.engines.every((r)=>typeof r==="string")){let r=e.engines.filter((a)=>Jt[a]),n=e.engines.filter((a)=>!Jt[a]);if(n.length>0)process.stderr.write(`[greedysearch] Warning: ignoring unknown engine(s) in ${V}: ${n.join(", ")}
|
|
267
|
+
[greedysearch] Available engines: ${Object.keys(Jt).join(", ")}
|
|
268
|
+
`);if(r.length>0)return r;process.stderr.write(`[greedysearch] Warning: no valid engines in ${V}, falling back to defaults: ${xt.join(", ")}
|
|
269
|
+
`)}}}catch{}return xt}function ca(){try{if(!st(Xt))fa(Xt,{recursive:!0});if(!st(V))oa(V,JSON.stringify({engines:xt,synthesizer:Qt},null,2)+`
|
|
270
|
+
`,"utf8")}catch{}}function wa(){try{if(st(V)){let t=er(V,"utf8"),e=JSON.parse(t);if(typeof e.synthesizer==="string"){let r=e.synthesizer.toLowerCase();if(He.includes(r))return r;process.stderr.write(`[greedysearch] Warning: unknown synthesizer "${e.synthesizer}" in ${V}
|
|
271
|
+
[greedysearch] Available synthesizers: ${He.join(", ")}
|
|
272
|
+
[greedysearch] Falling back to default: ${Qt}
|
|
273
|
+
`)}}}catch{}return Qt}function $a(){if(typeof globalThis.DOMMatrix>"u")globalThis.DOMMatrix=class{constructor(t=void 0){}multiplySelf(){return this}preMultiplySelf(){return this}translateSelf(){return this}scaleSelf(){return this}rotateSelf(){return this}};if(typeof globalThis.ImageData>"u")globalThis.ImageData=class{constructor(t=void 0,e=0,r=0){this.data=t,this.width=e,this.height=r}};if(typeof globalThis.Path2D>"u")globalThis.Path2D=class{constructor(t=void 0){}}}async function ma(){$a();let t=await import("pdf-parse"),e=t.PDFParse??t.default;if(!e)throw Error("pdf-parse did not export PDFParse");return e}async function ba(t,e){try{let r=new(await ma())({data:new Uint8Array(t)});await r.load();let n=await r.getText(),a=n.text?.trim();if(!a)return null;return{title:new URL(e).pathname.split("/").pop()||"Document.pdf",content:`## PDF Content (${n.total} pages)
|
|
274
|
+
|
|
275
|
+
${a}`,pages:n.total}}catch(r){return{error:r.message||String(r)}}}function ar(t="",e=240){let r=String(t).replaceAll(/\s+/g," ").trim();if(r.length<=e)return r;let n=r.slice(0,e),a=n.lastIndexOf(" ");return a>0?`${n.slice(0,a)}...`:`${n}...`}async function sa(t,e,r=8000){let n=Date.now();try{let a=(await R(["evalraw",t,"Page.getFrameTree","{}"]).then((m)=>JSON.parse(m)).catch(()=>null))?.frameTree?.frame?.id||void 0,l=await R(["evalraw",t,"Network.loadNetworkResource",JSON.stringify({frameId:a,url:e,options:{disableCache:!0,includeCredentials:!1}})],20000),i=JSON.parse(l).resource;if(!i?.success||!i.httpStatusCode)return{url:e,error:i?.netErrorName||i?.netError||"loadNetworkResource failed",source:"chrome",duration:Date.now()-n,needsFallback:!0};let f="";if(i.stream)try{let m=await R(["evalraw",t,"IO.read",JSON.stringify({handle:i.stream})],1e4);f=JSON.parse(m).data||"",await R(["evalraw",t,"IO.close",JSON.stringify({handle:i.stream})]).catch(()=>{})}catch{}if(!f||f.length<100)return{url:e,error:"Empty response body from Network.loadNetworkResource",source:"chrome",duration:Date.now()-n,needsFallback:!0};let g=de(i.httpStatusCode,f,e,e);if(g.blocked)return{url:e,status:i.httpStatusCode,error:`Blocked: ${g.reason}`,source:"chrome",duration:Date.now()-n,needsBrowser:!0};let o=Ve(f,e),p=Ze(o);if(!p.ok)return{url:e,status:i.httpStatusCode,error:`Low quality: ${p.reason}`,source:"chrome",duration:Date.now()-n,needsBrowser:!0};let h=q(o.markdown,r);return{url:e,finalUrl:e,status:i.httpStatusCode,contentType:"text/markdown",lastModified:"",publishedTime:o.publishedTime||"",byline:o.byline||"",siteName:o.siteName||"",lang:o.lang||"",title:o.title||e,snippet:o.excerpt,content:h,contentChars:h.length,source:"chrome",duration:Date.now()-n}}catch(a){return{url:e,error:a.message,source:"chrome",duration:Date.now()-n,needsFallback:!0}}}function lr(t){try{return new URL(t).pathname.toLowerCase().endsWith(".pdf")}catch{return!1}}async function _a(t,e=8000){let r=Ye(t);if(r.blocked)return{url:t,finalUrl:t,status:403,error:`Blocked: ${r.reason}`,source:"pdf-http"};let n=new AbortController,a=setTimeout(()=>n.abort(),20000),l=Date.now();try{let i=await fetch(t,{method:"GET",redirect:"follow",signal:n.signal,headers:Dn({accept:"application/pdf,application/octet-stream;q=0.9,*/*;q=0.5"})});clearTimeout(a);let f=i.headers.get("content-type")||"",g=i.url||t,o=Number.parseInt(i.headers.get("content-length")||"0",10);if(i.status>=400)return{url:t,finalUrl:g,status:i.status,error:`HTTP ${i.status}`,source:"pdf-http",duration:Date.now()-l};if(!f.toLowerCase().includes("application/pdf")&&!lr(g))return null;if(o>31457280)return{url:t,finalUrl:g,status:i.status,error:`PDF too large: ${o} bytes`,source:"pdf-http",duration:Date.now()-l};let p=Buffer.from(await i.arrayBuffer()),h=await ba(p,g);if(!h||h.error)return{url:t,finalUrl:g,status:i.status,error:h?.error||"PDF text extraction failed",source:"pdf-http",duration:Date.now()-l};let m=q(h.content,e);return{url:t,finalUrl:g,status:i.status,contentType:"application/pdf",lastModified:i.headers.get("last-modified")||"",title:h.title,snippet:ar(m,320),content:m,contentChars:m.length,pages:h.pages,source:"pdf-http",duration:Date.now()-l}}catch(i){return clearTimeout(a),{url:t,finalUrl:t,error:i.message||String(i),source:"pdf-http",duration:Date.now()-l}}}async function ir(t,e=8000){let r=Date.now();if(lr(t)){let a=await _a(t,e);if(a?.content||a?.status===403)return a}if(Yt(t)){let a=Yt(t);if(a&&(a.type==="root"||a.type==="tree"||a.type==="blob"&&!a.path?.includes("."))){let l=await Ln(t);if(l.ok){let i=q(l.content,e);return{url:t,finalUrl:t,status:200,contentType:"text/markdown",lastModified:"",title:l.title,snippet:i.slice(0,320),content:i,contentChars:i.length,source:"github-api",...l.tree&&{tree:l.tree},duration:Date.now()-r}}process.stderr.write(`[greedysearch] GitHub API fetch failed, trying HTTP: ${l.error}
|
|
276
|
+
`)}}if(Mn(t)?.type==="post"){process.stderr.write(`[greedysearch] Using Reddit JSON API for: ${t.slice(0,60)}...
|
|
277
277
|
`);let a=await Wn(t,e);if(a.ok){let l=q(a.markdown,e);return{url:t,finalUrl:a.finalUrl,status:a.status,contentType:"text/markdown",lastModified:a.lastModified||"",publishedTime:a.publishedTime||"",byline:a.byline||"",siteName:a.siteName||"",lang:a.lang||"",title:a.title,snippet:a.excerpt,content:l,contentChars:l.length,source:"reddit-api",duration:Date.now()-r}}process.stderr.write(`[greedysearch] Reddit API fetch failed, falling back to HTTP: ${a.error}
|
|
278
|
-
`)}let n=await
|
|
279
|
-
`),await
|
|
278
|
+
`)}let n=await kn(t,{timeoutMs:1e4});if(n.ok){let a=q(n.markdown,e);return{url:t,finalUrl:n.finalUrl,status:n.status,contentType:"text/markdown",lastModified:n.lastModified||"",publishedTime:n.publishedTime||"",byline:n.byline||"",siteName:n.siteName||"",lang:n.lang||"",title:n.title,snippet:n.excerpt,content:a,contentChars:a.length,source:"http",duration:Date.now()-r}}if(n.needsBrowser)try{let a=await St();try{let l=await sa(a,t,e);if(l.content&&l.content.length>100)return l}finally{await zt(a)}}catch{}return process.stderr.write(`[greedysearch] HTTP failed for ${t.slice(0,60)}, trying browser...
|
|
279
|
+
`),await Ea(t,e)}async function Ea(t,e=8000){let r=Date.now(),n;try{n=await St()}catch(a){return{url:t,title:"",content:null,snippet:"",contentChars:0,error:`openNewTab failed: ${a.message}`,source:"browser",duration:Date.now()-r}}try{await R(["nav",n,t],30000),await new Promise((f)=>setTimeout(f,800));let a=await R(["eval",n,String.raw`
|
|
280
280
|
(function(){
|
|
281
281
|
var el = document.querySelector('article, [role="main"], main, .post-content, .article-body, #content, .content');
|
|
282
282
|
var text = (el || document.body).innerText;
|
|
@@ -286,19 +286,19 @@ ${a}`,pages:n.total}}catch(r){return{error:r.message||String(r)}}}function rr(t=
|
|
|
286
286
|
url: location.href
|
|
287
287
|
});
|
|
288
288
|
})()
|
|
289
|
-
`]),l=JSON.parse(a),i=q(l.content,e);return{url:t,finalUrl:l.url||t,status:200,contentType:"text/plain",lastModified:"",title:l.title,snippet:
|
|
290
|
-
`);let i=Array(a.length),f=0,
|
|
291
|
-
`);let
|
|
292
|
-
`);else if(
|
|
293
|
-
`);
|
|
294
|
-
`)}}await Promise.all(Array.from({length:l},()=>
|
|
295
|
-
`),i}async function
|
|
289
|
+
`]),l=JSON.parse(a),i=q(l.content,e);return{url:t,finalUrl:l.url||t,status:200,contentType:"text/plain",lastModified:"",title:l.title,snippet:ar(i,320),content:i,contentChars:i.length,source:"browser",duration:Date.now()-r}}catch(a){return{url:t,title:"",content:null,snippet:"",contentChars:0,error:a.message,source:"browser",duration:Date.now()-r}}finally{await zt(n)}}async function ya(t,e=5,r=8000,n=Fe){let a=t.slice(0,e);if(a.length===0)return[];let l=Math.min(a.length,Math.max(1,Number.parseInt(String(n),10)||Fe));process.stderr.write(`[greedysearch] Fetching content from ${a.length} sources via HTTP (concurrency ${l})...
|
|
290
|
+
`);let i=Array(a.length),f=0,g=0;async function o(){while(!0){let w=f++;if(w>=a.length)return;let s=a[w],b=s.canonicalUrl||s.url;process.stderr.write(`[greedysearch] [${w+1}/${a.length}] Fetching: ${b.slice(0,60)}...
|
|
291
|
+
`);let c=await ir(b,r).catch((E)=>({url:b,title:"",content:null,snippet:"",contentChars:0,error:E.message,source:"error",duration:0}));if(i[w]={id:s.id,...c},c.content&&c.content.length>100)process.stderr.write(`[greedysearch] ✓ ${c.source}: ${c.content.length} chars
|
|
292
|
+
`);else if(c.error)process.stderr.write(`[greedysearch] ✗ ${c.error.slice(0,80)}
|
|
293
|
+
`);g+=1,process.stderr.write(`PROGRESS:fetch:${g}/${a.length}
|
|
294
|
+
`)}}await Promise.all(Array.from({length:l},()=>o()));let p=i.filter((w)=>w.content&&w.content.length>100),h=i.filter((w)=>w.source==="http").length,m=i.filter((w)=>w.source==="browser").length;return process.stderr.write(`[greedysearch] Fetched ${p.length}/${i.length} sources (HTTP: ${h}, Browser: ${m})
|
|
295
|
+
`),i}async function Da(t){let e=await St();await R(["list"]);try{await R(["nav",e,t],30000),await new Promise((n)=>setTimeout(n,800));let r=await R(["eval",e,String.raw`
|
|
296
296
|
(function(){
|
|
297
297
|
var el = document.querySelector('article, [role="main"], main, .post-content, .article-body, #content, .content');
|
|
298
298
|
var text = (el || document.body).innerText;
|
|
299
299
|
return text.replace(/\s+/g, ' ').trim();
|
|
300
300
|
})()
|
|
301
|
-
`]);return{url:t,content:r}}catch(r){return{url:t,content:null,error:r.message}}finally{await zt(e)}}var
|
|
301
|
+
`]);return{url:t,content:r}}catch(r){return{url:t,content:null,error:r.message}}finally{await zt(e)}}var $i,Ge,Ie="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",Je,yn,jn,Cn,Nn,Ke,An,Sn,zn,Wi,ze,Hi,Fi,Gi,Ii,dt,Y,Vt,Zt="gemini",Me,Gt,Ji,Yi,di,Vi,ue,Zi,Ki,Bi,Ui,Kt,d,Bt,Ut="gemini",We,It,Xi,xi,Qi,at,Si,zi,ui,qi,tf,ef,rf,qe,nf,af,lf,ff,R,cf,nr,wf,$f,mf,bf,Xt,V,xt,Qt="gemini",He,Jt,sf,_f,Fe;var or=ot(()=>{$i=bn(import.meta.url),Ge=new En({headingStyle:"atx",bulletListMarker:"-",codeBlockStyle:"fenced"});Ge.addRule("removeDataUrls",{filter:(t)=>t.tagName==="IMG"&&t.getAttribute("src")?.startsWith("data:"),replacement:()=>""});Je={"user-agent":Ie,accept:"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8","accept-language":"en-US,en;q=0.9","accept-encoding":"gzip, deflate, br","cache-control":"no-cache",pragma:"no-cache","sec-ch-ua":'"Chromium";v="122", "Not(A:Brand";v="24", "Google Chrome";v="122"',"sec-ch-ua-mobile":"?0","sec-ch-ua-platform":'"Windows"',"sec-fetch-dest":"document","sec-fetch-mode":"navigate","sec-fetch-site":"none","sec-fetch-user":"?1","upgrade-insecure-requests":"1"},yn=[/^localhost$/i,/^127\.\d+\.\d+\.\d+$/,/^0\.0\.0\.0$/,/^\[::1\]$/,/^10\./,/^172\.(1[6-9]|2\d|3[01])\./,/^192\.168\./,/^169\.254\./,/^fc00:/i,/^fe80:/i,/\.local$/i,/\.internal$/i,/\.localhost$/i];jn=["accounts.google.com","login.microsoftonline.com","login.live.com","auth0.com","okta.com","auth.mozilla.auth0.com","id.atlassian.com"],Cn=["login.","signin.","auth.","sso.","accounts.","idp."],Nn=["sign in to continue","log in to continue","authentication required","create an account to continue","subscribe to continue reading","members only"];Ke={"user-agent":"GreedySearch/1.0",accept:"application/vnd.github+json","x-github-api-version":"2022-11-28"};An={"user-agent":"GreedySearch/1.0 (Research Bot)",accept:"application/json"};Sn=In(Yn(import.meta.url)),zn=Jn(Sn,"..","bin","cdp.mjs");Wi=qn(process.env.GREEDY_SEARCH_PORT,9222),ze=(process.env.GREEDY_SEARCH_PROFILE_DIR||process.env.CDP_PROFILE_DIR||`${_t().replaceAll("\\","/")}/greedysearch-chrome-profile`).replaceAll("\\","/"),Hi=`${ze}/DevToolsActivePort`,Fi=`${_t().replaceAll("\\","/")}/cdp-pages.json`,Gi=process.env.GREEDY_SEARCH_MODE_FILE||`${_t().replaceAll("\\","/")}/greedysearch-chrome-mode`,Ii=`${_t().replaceAll("\\","/")}/greedysearch-visible-recovery.jsonl`,dt=Xe(Kn(),".dm"),Y=Xe(dt,"greedyconfig"),Vt=["perplexity","google","chatgpt"];ea();Me=["gemini","chatgpt"];Gt={perplexity:"perplexity.mjs",p:"perplexity.mjs",bing:"bing-copilot.mjs",b:"bing-copilot.mjs",google:"google-ai.mjs",g:"google-ai.mjs",gemini:"gemini.mjs",gem:"gemini.mjs",chatgpt:"chatgpt.mjs",gpt:"chatgpt.mjs","semantic-scholar":"semantic-scholar.mjs",semanticscholar:"semantic-scholar.mjs",s2:"semantic-scholar.mjs",logically:"logically.mjs",log:"logically.mjs"},Ji=ta(),Yi=ra(),di=Math.max(1,Number.parseInt(process.env.GREEDY_FETCH_CONCURRENCY||"5",10)||5);process.env.CDP_PROFILE_DIR=ze;Vi=na(process.env.GREEDY_SEARCH_PORT,9222),ue=(process.env.GREEDY_SEARCH_PROFILE_DIR||process.env.CDP_PROFILE_DIR||`${Et().replaceAll("\\","/")}/greedysearch-chrome-profile`).replaceAll("\\","/"),Zi=`${ue}/DevToolsActivePort`,Ki=`${Et().replaceAll("\\","/")}/cdp-pages.json`,Bi=process.env.GREEDY_SEARCH_MODE_FILE||`${Et().replaceAll("\\","/")}/greedysearch-chrome-mode`,Ui=`${Et().replaceAll("\\","/")}/greedysearch-visible-recovery.jsonl`,Kt=Qe(xn(),".dm"),d=Qe(Kt,"greedyconfig"),Bt=["perplexity","google","chatgpt"];la();We=["gemini","chatgpt"];It={perplexity:"perplexity.mjs",p:"perplexity.mjs",bing:"bing-copilot.mjs",b:"bing-copilot.mjs",google:"google-ai.mjs",g:"google-ai.mjs",gemini:"gemini.mjs",gem:"gemini.mjs",chatgpt:"chatgpt.mjs",gpt:"chatgpt.mjs","semantic-scholar":"semantic-scholar.mjs",semanticscholar:"semantic-scholar.mjs",s2:"semantic-scholar.mjs",logically:"logically.mjs",log:"logically.mjs"},Xi=aa(),xi=ia(),Qi=Math.max(1,Number.parseInt(process.env.GREEDY_FETCH_CONCURRENCY||"5",10)||5);process.env.CDP_PROFILE_DIR=ue;at=Bn().replaceAll("\\","/"),Si=`${at}/greedysearch-chrome-metadata.json`,zi=`${at}/greedysearch-chrome-launch.lock`,ui=`${at}/greedysearch-chrome.pid`,qi=`${at}/greedysearch-chrome-mode`,tf=`${at}/greedysearch-chrome-last-activity`,ef=Number.parseInt(process.env.GREEDY_SEARCH_IDLE_TIMEOUT_MINUTES||"5",10)||5,rf=import.meta.dirname||new URL(".",import.meta.url).pathname.replace(/^\/([A-Z]:)/,"$1"),qe=Fn().replaceAll("\\","/"),nf=`${qe}/greedysearch-chrome.pid`,af=`${qe}/greedysearch-chrome-last-activity`,lf=Number.parseInt(process.env.GREEDY_SEARCH_IDLE_TIMEOUT_MINUTES||"5",10)||5,ff=Number.parseInt(process.env.GREEDY_SEARCH_VISIBLE_IDLE_TIMEOUT_MINUTES||"60",10)||60,R=Se;cf=pa(process.env.GREEDY_SEARCH_PORT,9222),nr=(process.env.GREEDY_SEARCH_PROFILE_DIR||process.env.CDP_PROFILE_DIR||`${yt().replaceAll("\\","/")}/greedysearch-chrome-profile`).replaceAll("\\","/"),wf=`${nr}/DevToolsActivePort`,$f=`${yt().replaceAll("\\","/")}/cdp-pages.json`,mf=process.env.GREEDY_SEARCH_MODE_FILE||`${yt().replaceAll("\\","/")}/greedysearch-chrome-mode`,bf=`${yt().replaceAll("\\","/")}/greedysearch-visible-recovery.jsonl`,Xt=rr(ga(),".dm"),V=rr(Xt,"greedyconfig"),xt=["perplexity","google","chatgpt"];ca();He=["gemini","chatgpt"];Jt={perplexity:"perplexity.mjs",p:"perplexity.mjs",bing:"bing-copilot.mjs",b:"bing-copilot.mjs",google:"google-ai.mjs",g:"google-ai.mjs",gemini:"gemini.mjs",gem:"gemini.mjs",chatgpt:"chatgpt.mjs",gpt:"chatgpt.mjs","semantic-scholar":"semantic-scholar.mjs",semanticscholar:"semantic-scholar.mjs",s2:"semantic-scholar.mjs",logically:"logically.mjs",log:"logically.mjs"},sf=ha(),_f=wa(),Fe=Math.max(1,Number.parseInt(process.env.GREEDY_FETCH_CONCURRENCY||"5",10)||5);process.env.CDP_PROFILE_DIR=nr});var pr={};et(pr,{writeSourcesToFiles:()=>Ca});import{mkdirSync as va,writeFileSync as ka}from"node:fs";import{join as gr}from"node:path";function Ca(t,e=ja){return va(e,{recursive:!0}),t.map((r)=>{if(!r.content||r.content.length<10)return r;let n=String(r.id||"unknown").replace(/[^a-zA-Z0-9_-]/g,""),a=(r.canonicalUrl||r.url||"").replace(/^https?:\/\//,"").replace(/[^a-zA-Z0-9]/g,"-").slice(0,40),l=`${n}-${a}.md`,i=gr(e,l),f=`---
|
|
302
302
|
url: ${r.finalUrl||r.url}
|
|
303
303
|
title: ${r.title||""}
|
|
304
304
|
source: ${r.source||"unknown"}
|
|
@@ -306,38 +306,38 @@ status: ${r.status||""}
|
|
|
306
306
|
chars: ${r.contentChars||r.content.length}
|
|
307
307
|
---
|
|
308
308
|
|
|
309
|
-
`;
|
|
309
|
+
`;ka(i,f+r.content,"utf8");let{content:g,...o}=r;return{...o,contentPath:i,contentChars:r.contentChars||g.length}})}var ja;var hr=ot(()=>{ja=gr(process.cwd(),".dm","greedysearch-sources")});var ut={};et(ut,{parseGitHubUrl:()=>$r,fetchGitHubContent:()=>Pa});function $r(t){try{let e=new URL(t);if(!(e.hostname==="github.com"||e.hostname.endsWith(".github.com")))return null;let r=e.pathname.split("/").filter(Boolean);if(r.length<2)return null;let[n,a]=r;if(r.length===2)return{owner:n,repo:a,type:"root"};if(r.length>=4&&(r[2]==="blob"||r[2]==="tree")){let l=r[2],i=r[3],f=r.slice(4).join("/");return{owner:n,repo:a,type:l,ref:i,path:f}}return null}catch{return null}}async function B(t,e=1e4){let r=new AbortController,n=setTimeout(()=>r.abort(),e);try{let a=await fetch(`https://api.github.com${t}`,{headers:wr,signal:r.signal});if(clearTimeout(n),!a.ok)throw Error(`GitHub API ${a.status}: ${t}`);return await a.json()}catch(a){throw clearTimeout(n),a}}async function Na(t,e){try{let r=await B(`/repos/${t}/${e}/readme`);if(r.content&&r.encoding==="base64")return Buffer.from(r.content,"base64").toString("utf8");return""}catch{return""}}async function cr(t,e,r="HEAD",n=""){try{let a=await B(`/repos/${t}/${e}/git/ref/heads/${r==="HEAD"?"main":r}`).catch(()=>B(`/repos/${t}/${e}/git/ref/heads/master`).catch(()=>null)),l;if(a?.object?.sha)l=(await B(`/repos/${t}/${e}/git/commits/${a.object.sha}`)).tree.sha;else{let f=await B(`/repos/${t}/${e}`);l=(await B(`/repos/${t}/${e}/branches/${f.default_branch}`)).commit.commit.tree.sha}let i=(await B(`/repos/${t}/${e}/git/trees/${l}`)).tree||[];if(n)i=i.filter((f)=>f.path.startsWith(n));return i.slice(0,50).map((f)=>({path:f.path,type:f.type==="tree"?"dir":"file",size:f.size}))}catch{return[]}}async function Oa(t,e,r,n,a=1e4){let l=[`https://raw.githubusercontent.com/${t}/${e}/${r&&r!=="HEAD"?r:"main"}/${n}`,`https://raw.githubusercontent.com/${t}/${e}/master/${n}`];for(let i of l){let f=new AbortController,g=setTimeout(()=>f.abort(),a);try{let o=await fetch(i,{headers:{"user-agent":wr["user-agent"]},signal:f.signal});if(clearTimeout(g),o.ok)return await o.text()}catch{clearTimeout(g)}}return null}async function Pa(t){let e=$r(t);if(!e)return{ok:!1,error:"Not a valid GitHub URL"};let{owner:r,repo:n,type:a,ref:l,path:i}=e;try{if(a==="root"||a==="tree"&&!i){let[f,g,o]=await Promise.allSettled([B(`/repos/${r}/${n}`),Na(r,n),cr(r,n,l||"HEAD")]);if(f.status==="rejected")return{ok:!1,error:f.reason?.message||"Repo not found"};let p=f.value,h=g.status==="fulfilled"?g.value:"",m=o.status==="fulfilled"?o.value:[],w=p?.description?`
|
|
310
310
|
|
|
311
|
-
> ${p.description}`:"",
|
|
311
|
+
> ${p.description}`:"",s=p?.stargazers_count==null?"":` ⭐ ${p.stargazers_count}`,b=p?.language?` · ${p.language}`:"",c=`# ${r}/${n}${s}${b}${w}
|
|
312
312
|
|
|
313
|
-
`;if(h)
|
|
313
|
+
`;if(h)c+=h.slice(0,6000);else c+=`[No README found]
|
|
314
314
|
|
|
315
315
|
Files:
|
|
316
|
-
${m.map((
|
|
317
|
-
`)}`;return{ok:!0,title:`${r}/${n}`,content:
|
|
316
|
+
${m.map((E)=>` ${E.type==="dir"?"\uD83D\uDCC1":"\uD83D\uDCC4"} ${E.path}`).join(`
|
|
317
|
+
`)}`;return{ok:!0,title:`${r}/${n}`,content:c,tree:m.slice(0,30)}}if(a==="blob"&&i){let f=await Oa(r,n,l,i);if(f===null)return{ok:!1,error:`File not found: ${i}`};return{ok:!0,title:`${r}/${n}: ${i}`,content:f}}if(a==="tree"&&i){let f=await cr(r,n,l||"HEAD",i),g=f.map((o)=>` ${o.type==="dir"?"\uD83D\uDCC1":"\uD83D\uDCC4"} ${o.path}`).join(`
|
|
318
318
|
`);return{ok:!0,title:`${r}/${n}/${i}`,content:`[Directory: ${i}]
|
|
319
319
|
|
|
320
320
|
Files:
|
|
321
|
-
${
|
|
321
|
+
${g}`,tree:f}}return{ok:!1,error:"Unsupported GitHub URL type"}}catch(f){return{ok:!1,error:f.message}}}var wr;var qt=ot(()=>{wr={"user-agent":"GreedySearch/1.0",accept:"application/vnd.github+json","x-github-api-version":"2022-11-28"}});var te={};et(te,{truncateContent:()=>La,trimContentHeadTail:()=>Ta});function Ta(t,e=8000){if(!t||t.length<=e)return t;let r=`
|
|
322
322
|
|
|
323
323
|
[...content trimmed...]
|
|
324
324
|
|
|
325
325
|
`,n=e-r.length,a=Math.floor(n*0.75),l=n-a,i=a;while(i>a-100&&t[i]!==`
|
|
326
326
|
`)i--;if(i<=a-100)i=a;let f=t.length-l;while(f<t.length-l+100&&t[f]!==`
|
|
327
|
-
`)f++;if(f>=t.length-l+100)f=t.length-l;let
|
|
327
|
+
`)f++;if(f>=t.length-l+100)f=t.length-l;let g=t.slice(0,i).trimEnd(),o=t.slice(f).trimStart();return`${g}${r}${o}`}function La(t,e=8000){if(!t||t.length<=e)return t;let r=t.slice(0,e),n=r.lastIndexOf(" ");return n>0?`${r.slice(0,n)}...`:`${r}...`}var jr={};et(jr,{shouldUseBrowser:()=>Za,rewriteGitHubUrl:()=>yr,isPrivateUrl:()=>Er,fetchSourceHttp:()=>Ga,extractContent:()=>vr,detectBotBlock:()=>Dr,defaultFetchHeaders:()=>Fa,checkContentQuality:()=>kr});import{Readability as Aa}from"@mozilla/readability";import{JSDOM as Ma}from"jsdom";import Wa from"turndown";function Fa(t={}){return{..._r,...t}}function Er(t){try{let e=new URL(t),r=e.hostname.toLowerCase();for(let n of Ha)if(n.test(r))return{blocked:!0,reason:`Private/internal address: ${r}`};if(e.protocol==="file:")return{blocked:!0,reason:"File protocol not allowed"};return{blocked:!1}}catch(e){return{blocked:!0,reason:`Invalid URL: ${e.message}`}}}function yr(t){try{let e=new URL(t);if(!(e.hostname==="github.com"||e.hostname.endsWith(".github.com")))return t;let r=e.pathname.split("/").filter(Boolean);if(r.length<5)return t;let[n,a,l,i,...f]=r;if(l!=="blob")return t;let g=f.join("/");return`https://raw.githubusercontent.com/${n}/${a}/${i}/${g}`}catch{return t}}async function Ga(t,e={}){let r=Er(t);if(r.blocked)return{ok:!1,url:t,finalUrl:t,status:403,error:`Blocked: ${r.reason}`,needsBrowser:!1};let n=t;if(t=yr(t),t!==n)console.error(`[fetcher] Rewrote GitHub URL: ${n.slice(0,60)}... → raw.githubusercontent.com`);let{timeoutMs:a=15000,userAgent:l,signal:i}=e,f=new AbortController,g=setTimeout(()=>f.abort(),a);if(i)i.addEventListener("abort",()=>f.abort(),{once:!0});try{let o=await fetch(t,{method:"GET",headers:{..._r,"user-agent":l||sr},redirect:"follow",signal:f.signal});clearTimeout(g);let p=o.headers.get("content-type")||"",h=o.url,m=o.headers.get("last-modified")||"",w=!1;try{w=new URL(h).hostname.toLowerCase()==="raw.githubusercontent.com"}catch{}if(p.includes("text/plain")&&w){let C=await o.text();return{ok:!0,url:n,finalUrl:h,status:o.status,title:h.split("/").pop()||"GitHub File",byline:"",siteName:"GitHub",lang:"",publishedTime:m,lastModified:m,markdown:C,contentLength:C.length,excerpt:C.slice(0,300).replaceAll(/\n/g," "),needsBrowser:!1}}if(!p.includes("text/html")&&!p.includes("application/xhtml"))return{ok:!1,url:t,finalUrl:h,status:o.status,error:`Unsupported content type: ${p}`,needsBrowser:!1};let s=await o.text(),b=Dr(o.status,s,h,t);if(b.blocked)return{ok:!1,url:t,finalUrl:h,status:o.status,error:`Blocked: ${b.reason}`,needsBrowser:!0};let c=vr(s,h),E=kr(c);if(!E.ok)return{ok:!1,url:t,finalUrl:h,status:o.status,error:`Low quality content: ${E.reason}`,needsBrowser:!0};return{ok:!0,url:t,finalUrl:h,status:o.status,title:c.title,byline:c.byline,siteName:c.siteName,lang:c.lang,publishedTime:c.publishedTime||m,lastModified:m,markdown:c.markdown,excerpt:c.excerpt,contentLength:c.markdown.length,needsBrowser:!1}}catch(o){clearTimeout(g);let p=Va(o);return{ok:!1,url:t,finalUrl:t,status:0,error:o.message,needsBrowser:p}}}function Dr(t,e,r,n){let a=e.match(/<title[^>]*>([^<]*)<\/title>/i)?.[1]?.toLowerCase()||"",l=e.slice(0,30000).toLowerCase(),i=`${a} ${l}`;if(t===403||t===429||t===503)return{blocked:!0,reason:`HTTP ${t}`};let f=[{pattern:/class=["'][^"']*captcha["']|<div[^>]*id=["']captcha/i,reason:"captcha"},{pattern:/g-recaptcha|data-sitekey|i['"]m not a robot/i,reason:"captcha"},{pattern:/checking your browser.{0,100}please wait|cf-browser-verification/i,reason:"cloudflare challenge"},{pattern:/just a moment.{0,50}security check|ddos protection by cloudflare/i,reason:"cloudflare challenge"},{pattern:/unusual traffic.{0,50}from your computer network/i,reason:"unusual traffic"},{pattern:/bot detected|automated.{0,20}request/i,reason:"bot detection"},{pattern:/enable\s+javascript\s+to\s+view|javascript\s+is\s+required.{0,50}enabled/i,reason:"requires javascript"},{pattern:/access denied|accessdenied/i,reason:"access denied"},{pattern:/protected by anubis|anubis uses a proof-of-work/i,reason:"anubis challenge"}];for(let o of f)if(o.pattern.test(i))return{blocked:!0,reason:o.reason};let g=da(n,r,e);if(g)return{blocked:!0,reason:g};return{blocked:!1}}function da(t,e,r){try{let n=new URL(t),a=new URL(e);if(n.hostname.toLowerCase()===a.hostname.toLowerCase())return;let l=a.hostname.toLowerCase();if(Ia.some((f)=>l===f||l.endsWith(`.${f}`)))return`redirected to login (${a.hostname})`;if(Ja.some((f)=>l.startsWith(f)))return`redirected to login (${a.hostname})`;let i=r.slice(0,20000).toLowerCase();if(Ya.some((f)=>i.includes(f)))return`redirected to login page (${a.hostname})`}catch{}return}function Va(t){let e=t.message.toLowerCase();return e.includes("fetch failed")||e.includes("unable to verify")||e.includes("certificate")||e.includes("timeout")}function mr(t){let e=['meta[property="article:published_time"]','meta[name="article:published_time"]','meta[property="og:published_time"]','meta[name="publication_date"]','meta[name="date"]','meta[itemprop="datePublished"]','time[itemprop="datePublished"]','meta[name="DC.date"]'];for(let r of e){let n=t.querySelector(r),a=n?.getAttribute("content")||n?.getAttribute("datetime")||"";if(a)return a}return""}function vr(t,e){let r=new Ma(t,{url:e}).window.document,n=new Aa(r).parse();if(n&&n.content){let l=br.turndown(n.content).replaceAll(/\n{3,}/g,`
|
|
328
328
|
|
|
329
|
-
`).trim(),i=n.publishedTime||$r(r)||"";return{title:n.title||r.title||e,byline:n.byline||"",siteName:n.siteName||"",lang:n.lang||"",publishedTime:i,markdown:l,excerpt:l.slice(0,300).replaceAll(/\n/g," ")}}let a=r.body;if(a){let l=a.cloneNode(!0);l.querySelectorAll("script, style, nav, footer, header, aside").forEach((f)=>f.remove());let i=(l.textContent||"").replaceAll(/\s+/g," ").trim();return{title:r.title||e,byline:"",siteName:"",lang:"",publishedTime:$r(r),markdown:i,excerpt:i.slice(0,300)}}return{title:e,byline:"",siteName:"",lang:"",publishedTime:"",markdown:"",excerpt:""}}function vr(t){let e=t.markdown.trim().toLowerCase(),r=(t.title||"").toLowerCase();if(t.markdown.trim().length<100)return{ok:!1,reason:"content too short (< 100 chars)"};let n=e.toLowerCase(),a=[{check:()=>n.includes("loading")&&n.includes("please wait"),desc:"loading page"},{check:()=>n.includes("please ensure javascript is enabled"),desc:"requires javascript"},{check:()=>n.includes("enable javascript to view"),desc:"requires javascript"},{check:()=>n.includes("just a moment"),desc:"cloudflare challenge detected in content"},{check:()=>n.includes("verify you are human"),desc:"human verification"},{check:()=>n.includes("captcha required"),desc:"captcha in extracted content"},{check:()=>n.includes("access denied"),desc:"access denied in content"},{check:()=>/^\s{0,10}sign\s{1,5}in\s{0,10}$|^\s{0,10}log\s{1,5}in\s{0,10}$/im.test(e),desc:"login form only"}];for(let{check:l,desc:i}of a)if(l())return{ok:!1,reason:i};if(r.includes("just a moment")||r.includes("checking your browser"))return{ok:!1,reason:"cloudflare challenge page detected in title"};return{ok:!0}}function Ha(t){try{let e=new URL(t),r=e.hostname.toLowerCase(),n=e.pathname.toLowerCase();if(["react.dev","nextjs.org","vuejs.org","angular.io","svelte.dev","docs.expo.dev","tailwindcss.com","storybook.js.org"].some((a)=>r===a||r.endsWith(`.${a}`)))return!0;if(n.includes("/playground")||n.includes("/demo")||n.includes("/app"))return!0;if(e.hash&&e.hash.length>1)return!0;return!1}catch{return!1}}var cr,mr="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",br,Pa,Ma,Aa,Ra;var Dr=gt(()=>{cr=new Oa({headingStyle:"atx",bulletListMarker:"-",codeBlockStyle:"fenced"});cr.addRule("removeDataUrls",{filter:(t)=>t.tagName==="IMG"&&t.getAttribute("src")?.startsWith("data:"),replacement:()=>""});br={"user-agent":mr,accept:"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8","accept-language":"en-US,en;q=0.9","accept-encoding":"gzip, deflate, br","cache-control":"no-cache",pragma:"no-cache","sec-ch-ua":'"Chromium";v="122", "Not(A:Brand";v="24", "Google Chrome";v="122"',"sec-ch-ua-mobile":"?0","sec-ch-ua-platform":'"Windows"',"sec-fetch-dest":"document","sec-fetch-mode":"navigate","sec-fetch-site":"none","sec-fetch-user":"?1","upgrade-insecure-requests":"1"},Pa=[/^localhost$/i,/^127\.\d+\.\d+\.\d+$/,/^0\.0\.0\.0$/,/^\[::1\]$/,/^10\./,/^172\.(1[6-9]|2\d|3[01])\./,/^192\.168\./,/^169\.254\./,/^fc00:/i,/^fe80:/i,/\.local$/i,/\.internal$/i,/\.localhost$/i];Ma=["accounts.google.com","login.microsoftonline.com","login.live.com","auth0.com","okta.com","auth.mozilla.auth0.com","id.atlassian.com"],Aa=["login.","signin.","auth.","sso.","accounts.","idp."],Ra=["sign in to continue","log in to continue","authentication required","create an account to continue","subscribe to continue reading","members only"]});import{spawn as Fa}from"node:child_process";import{mkdirSync as te,writeFileSync as W}from"node:fs";import{join as E}from"node:path";import{fileURLToPath as Ya}from"node:url";var Zr=["fbclid","gclid","ref","ref_src","ref_url","source","utm_campaign","utm_content","utm_medium","utm_source","utm_term"],Br=["dev.to","hashnode.com","medium.com","reddit.com","stackoverflow.com","stackexchange.com","substack.com"],Kr=["arstechnica.com","techcrunch.com","theverge.com","venturebeat.com","wired.com","zdnet.com"],ke=["facebook.com","instagram.com","linkedin.com","pinterest.com","tiktok.com","twitter.com","x.com"];function T(t="",e=240){let r=String(t).replaceAll(/\s+/g," ").trim();if(r.length<=e)return r;let n=r.slice(0,e),a=n.lastIndexOf(" ");return a>0?`${n.slice(0,a)}...`:`${n}...`}function Tt(t=""){let e=T(t,180);if(!e)return"";if(/^https?:\/\//i.test(e))return"";let r=e.split(/\s+/).filter(Boolean).length,n=/[A-Z]/.test(e),a=/\d/.test(e);return e===e.toLowerCase()&&r<=4&&!n&&!a?"":e}function Ot(t="",e=""){let r=Tt(t),n=Tt(e);if(!n)return r;if(!r)return n;let a=/^https?:\/\//i.test(r),l=/^https?:\/\//i.test(n);if(a&&!l)return n;if(!a&&l)return r;return n.length>r.length?n:r}function M(t){if(!t)return null;try{let e=new URL(t);if(!["http:","https:"].includes(e.protocol))return null;if(e.hash="",e.hostname=e.hostname.toLowerCase(),e.protocol==="https:"&&e.port==="443"||e.protocol==="http:"&&e.port==="80")e.port="";for(let a of[...e.searchParams.keys()]){let l=a.toLowerCase();if(Zr.includes(l)||l.startsWith("utm_"))e.searchParams.delete(a)}e.searchParams.sort();let r=e.pathname.replace(/\/{1,10}$/,"")||"/";e.pathname=r;let n=e.toString();return r==="/"?n.replace(/\/$/,""):n}catch{return null}}function Ur(t){try{return new URL(t).hostname.toLowerCase().replace(/^www\./,"")}catch{return""}}function rt(t,e){return e.some((r)=>t===r||t.endsWith(`.${r}`))}function Pt(t,e="",r=""){let n=e.toLowerCase(),a=r.toLowerCase();if(t==="github.com"||t==="gitlab.com")return"repo";if(t==="arxiv.org"||t==="doi.org"||t==="semanticscholar.org"||t.endsWith(".semanticscholar.org")||a.includes("/paper/")||a.includes("/pdf/"))return"academic";if(rt(t,ke))return"social";if(rt(t,Br))return"community";if(rt(t,Kr))return"news";if(t.startsWith("docs.")||t.startsWith("developer.")||t.startsWith("developers.")||t.startsWith("api.")||n.includes("documentation")||n.includes("docs")||n.includes("reference")||a.includes("/docs/")||a.includes("/reference/")||a.includes("/api/"))return"official-docs";if(t.startsWith("blog.")||a.includes("/blog/"))return"maintainer-blog";return"website"}function Ir(t){switch(t){case"official-docs":return 5;case"repo":return 4;case"academic":return 4;case"maintainer-blog":return 3;case"website":return 2;case"community":return 1;case"news":return 0;case"social":return-6;default:return 0}}function Xr(t){let e=Object.values(t.perEngine||{}).map((r)=>r?.rank||99);return e.length?Math.min(...e):99}var xr=["reddit.com","news.ycombinator.com","lobste.rs"];function X(t){return t.smartScore*3+t.engineCount*5+Ir(t.sourceType)*2+Math.max(0,7-Xr(t))}function Qr(t){let e=t.toLowerCase(),r=[];if(e.includes("openai")||e.includes("gpt")||e.includes("chatgpt"))r.push("openai.com","platform.openai.com","help.openai.com");if(e.includes("anthropic")||e.includes("claude"))r.push("anthropic.com","docs.anthropic.com");if(e.includes("bun"))r.push("bun.sh","bun.com");if(e.includes("next.js")||e.includes("nextjs"))r.push("nextjs.org","vercel.com");if(e.includes("playwright"))r.push("playwright.dev");if(e.includes("supabase"))r.push("supabase.com","supabase.io");if(e.includes("prisma"))r.push("prisma.io");if(e.includes("tailwind"))r.push("tailwindcss.com");if(e.includes("vite"))r.push("vitejs.dev","vite.dev");if(e.includes("astro"))r.push("astro.build");if(e.includes("svelte"))r.push("svelte.dev");if(e.includes("solid"))r.push("solidjs.com");if(e.includes("vue")||e.includes("nuxt"))r.push("vuejs.org","nuxt.com");if(e.includes("react")||e.includes("react native"))r.push("react.dev","reactnative.dev");if(e.includes("angular"))r.push("angular.io","angular.dev");if(e.includes("node.js")||e.includes("nodejs"))r.push("nodejs.org","nodejs.dev","npmjs.com");if(/\bgo\b/.test(e)||e.includes("golang"))r.push("go.dev","golang.org","pkg.go.dev");if(e.includes("deno"))r.push("deno.land","deno.com");if(e.includes("fresh"))r.push("fresh.deno.dev");if(e.includes("typescript")||e.includes("ts"))r.push("typescriptlang.org");if(e.includes("python"))r.push("python.org","docs.python.org");if(e.includes("rust"))r.push("rust-lang.org","docs.rs","crates.io");if(e.includes("zig"))r.push("ziglang.org");if(e.includes("docker"))r.push("docker.com","docs.docker.com","hub.docker.com");if(e.includes("kubernetes")||e.includes("k8s"))r.push("kubernetes.io","k8s.io");if(e.includes("postgres")||e.includes("postgresql"))r.push("postgresql.org","neon.tech","supabase.com");if(e.includes("redis"))r.push("redis.io");if(e.includes("sqlite"))r.push("sqlite.org");if(e.includes("cloudflare"))r.push("developers.cloudflare.com","cloudflare.com");if(e.includes("vercel"))r.push("vercel.com","nextjs.org");if(e.includes("netlify"))r.push("netlify.com","docs.netlify.com");if(e.includes("stripe"))r.push("stripe.com","docs.stripe.com");if(e.includes("github"))r.push("github.com","docs.github.com");if(e.includes("gitlab"))r.push("gitlab.com","docs.gitlab.com");if(e.includes("aws"))r.push("aws.amazon.com","docs.aws.amazon.com");if(e.includes("azure"))r.push("azure.microsoft.com","learn.microsoft.com");if(e.includes("gcp")||e.includes("google cloud"))r.push("cloud.google.com","developers.google.com");if(e.includes("gemini")||e.includes("google ai"))r.push("ai.google.dev","developers.google.com");for(let n of ke){let a=n.replace(/\.com$/,"");if(e.includes(a))r.push(n)}return[...new Set(r)]}function _e(t,e){return t===e||t.endsWith(`.${e}`)}function Lt(t,e=""){let r=new Map,n=Object.keys(t||{}).filter((g)=>!g.startsWith("_")),a=Qr(e);for(let g of n){let p=t[g];if(!p?.sources)continue;for(let h=0;h<p.sources.length;h++){let m=p.sources[h],c=M(m.url);if(!c||c.length<10)continue;let y=Tt(m.title||""),b=Ur(c),w=Pt(b,y,c),k=0;if(a.some((N)=>_e(b,N)))k+=10;if(w==="official-docs")k+=3;let C=c.toLowerCase();if(/\/docs\/|\/documentation\/|\.dev\/|\/api\/|\/reference\//.test(C))k+=2;let _=a.some((N)=>_e(b,N));if(w==="social"&&!_)k-=20;if(a.length>0){if(rt(b,xr))k-=3;else if(w==="community"&&!rt(b,["stackoverflow.com","stackexchange.com"]))k-=1}let s=r.get(c)||{id:"",canonicalUrl:c,displayUrl:m.url||c,domain:b,title:"",engines:[],engineCount:0,perEngine:{},sourceType:w,isOfficial:w==="official-docs",smartScore:0};if(s.title=Ot(s.title,y),s.displayUrl=s.displayUrl||m.url||c,s.sourceType=s.sourceType||w,s.isOfficial=s.isOfficial||w==="official-docs",s.smartScore=Math.max(s.smartScore,k),!s.engines.includes(g))s.engines.push(g);s.perEngine[g]={rank:h+1,title:Ot(s.perEngine[g]?.title||"",y)},r.set(c,s)}}let l=Array.from(r.values()).map((g)=>({...g,engineCount:g.engines.length})),i=l.filter((g)=>g.sourceType!=="social"),f=l.filter((g)=>g.sourceType==="social");return i.sort((g,p)=>{let h=X(p)-X(g);if(h!==0)return h;return g.domain.localeCompare(p.domain)}),f.sort((g,p)=>{let h=X(p)-X(g);if(h!==0)return h;return g.domain.localeCompare(p.domain)}),[...i,...f].slice(0,12).map((g,p)=>({...g,id:`S${p+1}`,title:g.title||g.domain||g.canonicalUrl}))}function se(t,e){let r=new Map(e.map((n)=>[n.id,n]));return t.map((n)=>{let a=r.get(n.id);if(!a)return n;let l=Ot(n.title,a.title||"");return{...n,title:l||n.title,fetch:{attempted:!0,ok:!a.error&&a.contentChars>100,status:a.status||null,finalUrl:a.finalUrl||a.url||n.canonicalUrl,contentType:a.contentType||"",lastModified:a.lastModified||"",publishedTime:a.publishedTime||"",byline:a.byline||"",siteName:a.siteName||"",lang:a.lang||"",title:a.title||"",snippet:a.snippet||"",contentChars:a.contentChars||0,source:a.source||"unknown",duration:a.duration||0,error:a.error||""}}})}import{existsSync as ot,mkdirSync as zr,readFileSync as ve,writeFileSync as Sr}from"node:fs";import{homedir as ur}from"node:os";import{join as je}from"node:path";import{tmpdir as pt}from"node:os";var ht=`${pt().replaceAll("\\","/")}/greedysearch-chrome-profile`,Rl=`${ht}/DevToolsActivePort`,Jl=`${pt().replaceAll("\\","/")}/cdp-pages.json`,dl=`${pt().replaceAll("\\","/")}/greedysearch-chrome-mode`,Hl=`${pt().replaceAll("\\","/")}/greedysearch-visible-recovery.jsonl`,Mt=je(ur(),".dm"),Y=je(Mt,"greedyconfig"),At=["perplexity","google","chatgpt"],Rt="gemini";function qr(){try{if(ot(Y)){let t=ve(Y,"utf8"),e=JSON.parse(t);if(Array.isArray(e.engines)&&e.engines.length>0&&e.engines.every((r)=>typeof r==="string")){let r=e.engines.filter((a)=>Wt[a]),n=e.engines.filter((a)=>!Wt[a]);if(n.length>0)process.stderr.write(`[greedysearch] Warning: ignoring unknown engine(s) in ${Y}: ${n.join(", ")}
|
|
330
|
-
[greedysearch] Available engines: ${Object.keys(
|
|
331
|
-
`);if(r.length>0)return r;process.stderr.write(`[greedysearch] Warning: no valid engines in ${
|
|
332
|
-
`)}}}catch{}return
|
|
333
|
-
`,"utf8")}catch{}}
|
|
329
|
+
`).trim(),i=n.publishedTime||mr(r)||"";return{title:n.title||r.title||e,byline:n.byline||"",siteName:n.siteName||"",lang:n.lang||"",publishedTime:i,markdown:l,excerpt:l.slice(0,300).replaceAll(/\n/g," ")}}let a=r.body;if(a){let l=a.cloneNode(!0);l.querySelectorAll("script, style, nav, footer, header, aside").forEach((f)=>f.remove());let i=(l.textContent||"").replaceAll(/\s+/g," ").trim();return{title:r.title||e,byline:"",siteName:"",lang:"",publishedTime:mr(r),markdown:i,excerpt:i.slice(0,300)}}return{title:e,byline:"",siteName:"",lang:"",publishedTime:"",markdown:"",excerpt:""}}function kr(t){let e=t.markdown.trim().toLowerCase(),r=(t.title||"").toLowerCase();if(t.markdown.trim().length<100)return{ok:!1,reason:"content too short (< 100 chars)"};let n=e.toLowerCase(),a=[{check:()=>n.includes("loading")&&n.includes("please wait"),desc:"loading page"},{check:()=>n.includes("please ensure javascript is enabled"),desc:"requires javascript"},{check:()=>n.includes("enable javascript to view"),desc:"requires javascript"},{check:()=>n.includes("just a moment"),desc:"cloudflare challenge detected in content"},{check:()=>n.includes("verify you are human"),desc:"human verification"},{check:()=>n.includes("captcha required"),desc:"captcha in extracted content"},{check:()=>n.includes("access denied"),desc:"access denied in content"},{check:()=>/^\s{0,10}sign\s{1,5}in\s{0,10}$|^\s{0,10}log\s{1,5}in\s{0,10}$/im.test(e),desc:"login form only"}];for(let{check:l,desc:i}of a)if(l())return{ok:!1,reason:i};if(r.includes("just a moment")||r.includes("checking your browser"))return{ok:!1,reason:"cloudflare challenge page detected in title"};return{ok:!0}}function Za(t){try{let e=new URL(t),r=e.hostname.toLowerCase(),n=e.pathname.toLowerCase();if(["react.dev","nextjs.org","vuejs.org","angular.io","svelte.dev","docs.expo.dev","tailwindcss.com","storybook.js.org"].some((a)=>r===a||r.endsWith(`.${a}`)))return!0;if(n.includes("/playground")||n.includes("/demo")||n.includes("/app"))return!0;if(e.hash&&e.hash.length>1)return!0;return!1}catch{return!1}}var br,sr="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",_r,Ha,Ia,Ja,Ya;var Cr=ot(()=>{br=new Wa({headingStyle:"atx",bulletListMarker:"-",codeBlockStyle:"fenced"});br.addRule("removeDataUrls",{filter:(t)=>t.tagName==="IMG"&&t.getAttribute("src")?.startsWith("data:"),replacement:()=>""});_r={"user-agent":sr,accept:"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8","accept-language":"en-US,en;q=0.9","accept-encoding":"gzip, deflate, br","cache-control":"no-cache",pragma:"no-cache","sec-ch-ua":'"Chromium";v="122", "Not(A:Brand";v="24", "Google Chrome";v="122"',"sec-ch-ua-mobile":"?0","sec-ch-ua-platform":'"Windows"',"sec-fetch-dest":"document","sec-fetch-mode":"navigate","sec-fetch-site":"none","sec-fetch-user":"?1","upgrade-insecure-requests":"1"},Ha=[/^localhost$/i,/^127\.\d+\.\d+\.\d+$/,/^0\.0\.0\.0$/,/^\[::1\]$/,/^10\./,/^172\.(1[6-9]|2\d|3[01])\./,/^192\.168\./,/^169\.254\./,/^fc00:/i,/^fe80:/i,/\.local$/i,/\.internal$/i,/\.localhost$/i];Ia=["accounts.google.com","login.microsoftonline.com","login.live.com","auth0.com","okta.com","auth.mozilla.auth0.com","id.atlassian.com"],Ja=["login.","signin.","auth.","sso.","accounts.","idp."],Ya=["sign in to continue","log in to continue","authentication required","create an account to continue","subscribe to continue reading","members only"]});import{spawn as Ka}from"node:child_process";import{mkdirSync as ee,writeFileSync as L}from"node:fs";import{join as j}from"node:path";import{fileURLToPath as Ba}from"node:url";var Kr=["fbclid","gclid","ref","ref_src","ref_url","source","utm_campaign","utm_content","utm_medium","utm_source","utm_term"],Br=["dev.to","hashnode.com","medium.com","reddit.com","stackoverflow.com","stackexchange.com","substack.com"],Ur=["arstechnica.com","techcrunch.com","theverge.com","venturebeat.com","wired.com","zdnet.com"],ye=["facebook.com","instagram.com","linkedin.com","pinterest.com","tiktok.com","twitter.com","x.com"];function O(t="",e=240){let r=String(t).replaceAll(/\s+/g," ").trim();if(r.length<=e)return r;let n=r.slice(0,e),a=n.lastIndexOf(" ");return a>0?`${n.slice(0,a)}...`:`${n}...`}function Ot(t=""){let e=O(t,180);if(!e)return"";if(/^https?:\/\//i.test(e))return"";let r=e.split(/\s+/).filter(Boolean).length,n=/[A-Z]/.test(e),a=/\d/.test(e);return e===e.toLowerCase()&&r<=4&&!n&&!a?"":e}function Pt(t="",e=""){let r=Ot(t),n=Ot(e);if(!n)return r;if(!r)return n;let a=/^https?:\/\//i.test(r),l=/^https?:\/\//i.test(n);if(a&&!l)return n;if(!a&&l)return r;return n.length>r.length?n:r}function A(t){if(!t)return null;try{let e=new URL(t);if(!["http:","https:"].includes(e.protocol))return null;if(e.hash="",e.hostname=e.hostname.toLowerCase(),e.protocol==="https:"&&e.port==="443"||e.protocol==="http:"&&e.port==="80")e.port="";for(let a of[...e.searchParams.keys()]){let l=a.toLowerCase();if(Kr.includes(l)||l.startsWith("utm_"))e.searchParams.delete(a)}e.searchParams.sort();let r=e.pathname.replace(/\/{1,10}$/,"")||"/";e.pathname=r;let n=e.toString();return r==="/"?n.replace(/\/$/,""):n}catch{return null}}function Xr(t){try{return new URL(t).hostname.toLowerCase().replace(/^www\./,"")}catch{return""}}function rt(t,e){return e.some((r)=>t===r||t.endsWith(`.${r}`))}function Rt(t,e="",r=""){let n=e.toLowerCase(),a=r.toLowerCase();if(t==="github.com"||t==="gitlab.com")return"repo";if(t==="arxiv.org"||t==="doi.org"||t==="semanticscholar.org"||t.endsWith(".semanticscholar.org")||a.includes("/paper/")||a.includes("/pdf/"))return"academic";if(rt(t,ye))return"social";if(rt(t,Br))return"community";if(rt(t,Ur))return"news";if(t.startsWith("docs.")||t.startsWith("developer.")||t.startsWith("developers.")||t.startsWith("api.")||n.includes("documentation")||n.includes("docs")||n.includes("reference")||a.includes("/docs/")||a.includes("/reference/")||a.includes("/api/"))return"official-docs";if(t.startsWith("blog.")||a.includes("/blog/"))return"maintainer-blog";return"website"}function xr(t){switch(t){case"official-docs":return 5;case"repo":return 4;case"academic":return 4;case"maintainer-blog":return 3;case"website":return 2;case"community":return 1;case"news":return 0;case"social":return-6;default:return 0}}function Qr(t){let e=Object.values(t.perEngine||{}).map((r)=>r?.rank||99);return e.length?Math.min(...e):99}var Sr=["reddit.com","news.ycombinator.com","lobste.rs"];function X(t){return t.smartScore*3+t.engineCount*5+xr(t.sourceType)*2+Math.max(0,7-Qr(t))}function zr(t){let e=t.toLowerCase(),r=[];if(e.includes("openai")||e.includes("gpt")||e.includes("chatgpt"))r.push("openai.com","platform.openai.com","help.openai.com");if(e.includes("anthropic")||e.includes("claude"))r.push("anthropic.com","docs.anthropic.com");if(e.includes("bun"))r.push("bun.sh","bun.com");if(e.includes("next.js")||e.includes("nextjs"))r.push("nextjs.org","vercel.com");if(e.includes("playwright"))r.push("playwright.dev");if(e.includes("supabase"))r.push("supabase.com","supabase.io");if(e.includes("prisma"))r.push("prisma.io");if(e.includes("tailwind"))r.push("tailwindcss.com");if(e.includes("vite"))r.push("vitejs.dev","vite.dev");if(e.includes("astro"))r.push("astro.build");if(e.includes("svelte"))r.push("svelte.dev");if(e.includes("solid"))r.push("solidjs.com");if(e.includes("vue")||e.includes("nuxt"))r.push("vuejs.org","nuxt.com");if(e.includes("react")||e.includes("react native"))r.push("react.dev","reactnative.dev");if(e.includes("angular"))r.push("angular.io","angular.dev");if(e.includes("node.js")||e.includes("nodejs"))r.push("nodejs.org","nodejs.dev","npmjs.com");if(/\bgo\b/.test(e)||e.includes("golang"))r.push("go.dev","golang.org","pkg.go.dev");if(e.includes("deno"))r.push("deno.land","deno.com");if(e.includes("fresh"))r.push("fresh.deno.dev");if(e.includes("typescript")||e.includes("ts"))r.push("typescriptlang.org");if(e.includes("python"))r.push("python.org","docs.python.org");if(e.includes("rust"))r.push("rust-lang.org","docs.rs","crates.io");if(e.includes("zig"))r.push("ziglang.org");if(e.includes("docker"))r.push("docker.com","docs.docker.com","hub.docker.com");if(e.includes("kubernetes")||e.includes("k8s"))r.push("kubernetes.io","k8s.io");if(e.includes("postgres")||e.includes("postgresql"))r.push("postgresql.org","neon.tech","supabase.com");if(e.includes("redis"))r.push("redis.io");if(e.includes("sqlite"))r.push("sqlite.org");if(e.includes("cloudflare"))r.push("developers.cloudflare.com","cloudflare.com");if(e.includes("vercel"))r.push("vercel.com","nextjs.org");if(e.includes("netlify"))r.push("netlify.com","docs.netlify.com");if(e.includes("stripe"))r.push("stripe.com","docs.stripe.com");if(e.includes("github"))r.push("github.com","docs.github.com");if(e.includes("gitlab"))r.push("gitlab.com","docs.gitlab.com");if(e.includes("aws"))r.push("aws.amazon.com","docs.aws.amazon.com");if(e.includes("azure"))r.push("azure.microsoft.com","learn.microsoft.com");if(e.includes("gcp")||e.includes("google cloud"))r.push("cloud.google.com","developers.google.com");if(e.includes("gemini")||e.includes("google ai"))r.push("ai.google.dev","developers.google.com");for(let n of ye){let a=n.replace(/\.com$/,"");if(e.includes(a))r.push(n)}return[...new Set(r)]}function Ee(t,e){return t===e||t.endsWith(`.${e}`)}function Tt(t,e=""){let r=new Map,n=Object.keys(t||{}).filter((o)=>!o.startsWith("_")),a=zr(e);for(let o of n){let p=t[o];if(!p?.sources)continue;for(let h=0;h<p.sources.length;h++){let m=p.sources[h],w=A(m.url);if(!w||w.length<10)continue;let s=Ot(m.title||""),b=Xr(w),c=Rt(b,s,w),E=0;if(a.some((N)=>Ee(b,N)))E+=10;if(c==="official-docs")E+=3;let C=w.toLowerCase();if(/\/docs\/|\/documentation\/|\.dev\/|\/api\/|\/reference\//.test(C))E+=2;let _=a.some((N)=>Ee(b,N));if(c==="social"&&!_)E-=20;if(a.length>0){if(rt(b,Sr))E-=3;else if(c==="community"&&!rt(b,["stackoverflow.com","stackexchange.com"]))E-=1}let y=r.get(w)||{id:"",canonicalUrl:w,displayUrl:m.url||w,domain:b,title:"",engines:[],engineCount:0,perEngine:{},sourceType:c,isOfficial:c==="official-docs",smartScore:0};if(y.title=Pt(y.title,s),y.displayUrl=y.displayUrl||m.url||w,y.sourceType=y.sourceType||c,y.isOfficial=y.isOfficial||c==="official-docs",y.smartScore=Math.max(y.smartScore,E),!y.engines.includes(o))y.engines.push(o);y.perEngine[o]={rank:h+1,title:Pt(y.perEngine[o]?.title||"",s)},r.set(w,y)}}let l=Array.from(r.values()).map((o)=>({...o,engineCount:o.engines.length})),i=l.filter((o)=>o.sourceType!=="social"),f=l.filter((o)=>o.sourceType==="social");return i.sort((o,p)=>{let h=X(p)-X(o);if(h!==0)return h;return o.domain.localeCompare(p.domain)}),f.sort((o,p)=>{let h=X(p)-X(o);if(h!==0)return h;return o.domain.localeCompare(p.domain)}),[...i,...f].slice(0,12).map((o,p)=>({...o,id:`S${p+1}`,title:o.title||o.domain||o.canonicalUrl}))}function De(t,e){let r=new Map(e.map((n)=>[n.id,n]));return t.map((n)=>{let a=r.get(n.id);if(!a)return n;let l=Pt(n.title,a.title||"");return{...n,title:l||n.title,fetch:{attempted:!0,ok:!a.error&&a.contentChars>100,status:a.status||null,finalUrl:a.finalUrl||a.url||n.canonicalUrl,contentType:a.contentType||"",lastModified:a.lastModified||"",publishedTime:a.publishedTime||"",byline:a.byline||"",siteName:a.siteName||"",lang:a.lang||"",title:a.title||"",snippet:a.snippet||"",contentChars:a.contentChars||0,source:a.source||"unknown",duration:a.duration||0,error:a.error||""}}})}import{existsSync as gt,mkdirSync as ur,readFileSync as ve,writeFileSync as qr}from"node:fs";import{homedir as tn}from"node:os";import{join as ke}from"node:path";import{tmpdir as pt}from"node:os";function en(t,e=9222){let r=Number.parseInt(String(t??""),10);return Number.isInteger(r)&&r>1024&&r<65535?r:e}var Yl=en(process.env.GREEDY_SEARCH_PORT,9222),ht=(process.env.GREEDY_SEARCH_PROFILE_DIR||process.env.CDP_PROFILE_DIR||`${pt().replaceAll("\\","/")}/greedysearch-chrome-profile`).replaceAll("\\","/"),dl=`${ht}/DevToolsActivePort`,Vl=`${pt().replaceAll("\\","/")}/cdp-pages.json`,Zl=process.env.GREEDY_SEARCH_MODE_FILE||`${pt().replaceAll("\\","/")}/greedysearch-chrome-mode`,Kl=`${pt().replaceAll("\\","/")}/greedysearch-visible-recovery.jsonl`,At=ke(tn(),".dm"),J=ke(At,"greedyconfig"),Mt=["perplexity","google","chatgpt"],Wt="gemini";function rn(){try{if(gt(J)){let t=ve(J,"utf8"),e=JSON.parse(t);if(Array.isArray(e.engines)&&e.engines.length>0&&e.engines.every((r)=>typeof r==="string")){let r=e.engines.filter((a)=>Lt[a]),n=e.engines.filter((a)=>!Lt[a]);if(n.length>0)process.stderr.write(`[greedysearch] Warning: ignoring unknown engine(s) in ${J}: ${n.join(", ")}
|
|
330
|
+
[greedysearch] Available engines: ${Object.keys(Lt).join(", ")}
|
|
331
|
+
`);if(r.length>0)return r;process.stderr.write(`[greedysearch] Warning: no valid engines in ${J}, falling back to defaults: ${Mt.join(", ")}
|
|
332
|
+
`)}}}catch{}return Mt}function nn(){try{if(!gt(At))ur(At,{recursive:!0});if(!gt(J))qr(J,JSON.stringify({engines:Mt,synthesizer:Wt},null,2)+`
|
|
333
|
+
`,"utf8")}catch{}}nn();var nt=["gemini","chatgpt"];function an(){try{if(gt(J)){let t=ve(J,"utf8"),e=JSON.parse(t);if(typeof e.synthesizer==="string"){let r=e.synthesizer.toLowerCase();if(nt.includes(r))return r;process.stderr.write(`[greedysearch] Warning: unknown synthesizer "${e.synthesizer}" in ${J}
|
|
334
334
|
[greedysearch] Available synthesizers: ${nt.join(", ")}
|
|
335
|
-
[greedysearch] Falling back to default: ${
|
|
336
|
-
`)}}}catch{}return
|
|
337
|
-
`)e+="\\n";else if(r&&a==="\r")e+="\\r";else if(r&&a==="\t")e+="\\t";else e+=a}return e}function
|
|
338
|
-
`||n==="\r"){if(!r)e+=" ";r=!0}else e+=n,r=!1;return e.trim()}function
|
|
339
|
-
`)}function
|
|
340
|
-
`),{score:a.length>0?a[a.length-1]:5,coverage:{},knowledgeGaps:[],shouldContinue:!0,nextActions:[],terminationReason:null,evaluationError:l.message}}}function
|
|
335
|
+
[greedysearch] Falling back to default: ${Wt}
|
|
336
|
+
`)}}}catch{}return Wt}var Lt={perplexity:"perplexity.mjs",p:"perplexity.mjs",bing:"bing-copilot.mjs",b:"bing-copilot.mjs",google:"google-ai.mjs",g:"google-ai.mjs",gemini:"gemini.mjs",gem:"gemini.mjs",chatgpt:"chatgpt.mjs",gpt:"chatgpt.mjs","semantic-scholar":"semantic-scholar.mjs",semanticscholar:"semantic-scholar.mjs",s2:"semantic-scholar.mjs",logically:"logically.mjs",log:"logically.mjs"},je=rn(),Ht=je,Bl=an(),Ul=Math.max(1,Number.parseInt(process.env.GREEDY_FETCH_CONCURRENCY||"5",10)||5);process.env.CDP_PROFILE_DIR=ht;function ln(t){let e="",r=!1,n=!1;for(let a of String(t)){if(n){e+=a,n=!1;continue}if(a==="\\"){e+=a,n=!0;continue}if(a==='"'){r=!r,e+=a;continue}if(r&&a===`
|
|
337
|
+
`)e+="\\n";else if(r&&a==="\r")e+="\\r";else if(r&&a==="\t")e+="\\t";else e+=a}return e}function ct(t){if(!t)return null;let e=String(t).trim(),r=e.indexOf("BEGIN_JSON"),n=e.indexOf("END_JSON");if(r!==-1&&n!==-1&&r<n)e=e.slice(r+10,n).trim();else{let f=e.indexOf("{");if(f>0)e=e.slice(f)}let a=[e,e.replace(/^```json\s*/i,"").replace(/^```\s*/i,"").replace(/```$/i,"").trim()],l=e.indexOf("{"),i=e.lastIndexOf("}");if(l!==-1&&i!==-1&&l<i)a.push(e.slice(l,i+1));for(let f of[...a]){let g=ln(f);if(g!==f)a.push(g)}for(let f of a)try{return JSON.parse(f)}catch{}return null}function Ft(t){let e=String(t??"").trim();if(!e)return null;let r=[e],n=e.indexOf("{"),a=e.lastIndexOf("}");if(n>=0&&a>n)r.push(e.slice(n,a+1));for(let l of r)try{let i=JSON.parse(l);if(i&&typeof i==="object"&&!Array.isArray(i))return i}catch{}return null}function Ce(t,e={}){let r=Ft(t);if(!r)return null;return{...r,_process:{...r._process&&typeof r._process==="object"?r._process:{},status:"partial",...e}}}import{spawn as hn}from"node:child_process";import{basename as fn}from"node:path";function wt(t=process.env,e=process.execPath){let r=t.GREEDY_SEARCH_NODE||t.NODE_BINARY||t.NODE;if(r?.trim())return r.trim();let n=fn(e||"").toLowerCase();if(n==="node"||n==="node.exe")return e;return"node"}import{existsSync as Ne}from"node:fs";import{dirname as on,join as $t}from"node:path";import{fileURLToPath as gn}from"node:url";function Oe(t){return gn(new URL(".",t)).replace(/^\/([A-Z]:)/,"$1")}function pn(t,e=Ne){let r=new Set;for(let n of t.filter(Boolean)){if(r.has(n))continue;if(r.add(n),e(n))return n}return t.filter(Boolean).at(-1)}function Pe(t,{moduleDir:e,entrypoint:r=process.argv[1],env:n=process.env,exists:a=Ne}={}){let l=r?on(r):null,i=n.GREEDY_SEARCH_EXTENSION_DIR?.trim()||null,f=[e?$t(e,"..","..","extractors",t):null,e?$t(e,"..","extractors",t):null,i?$t(i,"extractors",t):null,l?$t(l,"..","extractors",t):null];return pn(f,a)}var cn=Oe(import.meta.url),wn={gemini:"gemini.mjs",chatgpt:"chatgpt.mjs"};function $n(t="gemini"){let e=String(t||"gemini").toLowerCase();if(e==="gem")return"gemini";if(e==="gpt")return"chatgpt";return e}async function mn(t,e,{tabPrefix:r=null,timeoutMs:n=180000,visible:a=null}={}){let l=$n(t),i=wn[l];if(!i||!nt.includes(l))throw Error(`Unsupported synthesizer "${t}". Supported: ${nt.join(", ")}`);let f=Pe(i,{moduleDir:cn});return new Promise((g,o)=>{let p=r?["--tab",String(r)]:[],h={...process.env,CDP_PROFILE_DIR:ht};if(a!==!0)delete h.GREEDY_SEARCH_VISIBLE,delete h.GREEDY_SEARCH_ALWAYS_VISIBLE;else h.GREEDY_SEARCH_VISIBLE="1",h.GREEDY_SEARCH_ALWAYS_VISIBLE="1";let m=hn(wt(),[f,"--stdin",...p],{stdio:["pipe","pipe","pipe"],env:h});m.stdin.write(e),m.stdin.end();let w="",s="";m.stdout.on("data",(c)=>w+=c),m.stderr.on("data",(c)=>s+=c);let b=setTimeout(()=>{m.kill(),o(Error(`${l} prompt timed out after ${n/1000}s`))},n);m.on("close",(c)=>{if(clearTimeout(b),c!==0){o(Error(s.trim()||`${l} extractor failed`));return}try{g(JSON.parse(w.trim()))}catch{o(Error(`bad JSON from ${l}: ${w.slice(0,100)}`))}})})}async function x(t,e={}){return mn("gemini",t,e)}var Ua=Ba(new URL(".",import.meta.url)).replace(/^\/([A-Z]:)/,"$1"),Xa=j(Ua,"..","..","bin","search.mjs"),xa=j(process.cwd(),".dm","greedysearch-research");function Qa(t){return String(t||"research").toLowerCase().replaceAll(/[^a-z0-9]+/g,"-").replaceAll(/^-|-$/g,"").slice(0,60)||"research"}function Z(t,e=1/0){let r=new Set,n=[];for(let a of t||[]){let l=O(String(a||""),1000);if(!l||r.has(l))continue;if(r.add(l),n.push(l),n.length>=e)break}return n}async function Sa(...t){let{fetchMultipleSources:e}=await Promise.resolve().then(() => (or(),fr));return e(...t)}async function ie(...t){let{writeSourcesToFiles:e}=await Promise.resolve().then(() => (hr(),pr));return e(...t)}function za({breadth:t=3,iterations:e=2,maxSources:r}){let n=re(t,1,5,3),a=re(e,1,3,2),l=re(r??Math.max(5,n*a*2),3,12,8);return{breadth:n,iterations:a,maxSources:l}}function re(t,e,r,n){let a=Number.parseInt(String(t??""),10);if(!Number.isFinite(a))return n;return Math.min(r,Math.max(e,a))}function ua(t,e,r,{expand:n=!0,includeOriginal:a=!0,exclude:l=[]}={}){let i=Array.isArray(t?.queries)?t.queries:[],f=[],g=new Set([...l].map((o)=>M(o).toLowerCase()));for(let o of i){let p=typeof o==="string"?o:o?.query,h=typeof o==="string"?"":o?.researchGoal||"";ne(f,p,h,{exclude:g})}if(a)ne(f,e,"Original user query",{prepend:!0,exclude:g});if(n){let o=[{query:`${e} official docs GitHub`,researchGoal:"Find primary project docs, repository details, and maintainer claims."},{query:`${e} benchmarks limitations compatibility`,researchGoal:"Validate performance claims and uncover unsupported APIs or caveats."},{query:`${e} alternatives comparison production use cases`,researchGoal:"Compare against conventional headless browsers and identify when to choose it."},{query:`${e} anti bot detection Cloudflare screenshots visual rendering`,researchGoal:"Check automation risks, rendering gaps, screenshots, and bot-detection behavior."}];for(let p of o){if(f.length>=r)break;ne(f,p.query,p.researchGoal,{exclude:g})}}return f.slice(0,r)}function ne(t,e,r="",{prepend:n=!1,exclude:a=new Set}={}){if(!e||typeof e!=="string")return;let l=M(e);if(!l||a.has(l.toLowerCase())||t.some((f)=>f.query.toLowerCase()===l.toLowerCase()))return;let i={query:l,researchGoal:O(r,320)};if(n)t.unshift(i);else t.push(i)}function M(t){return tl(qa(String(t)))}function qa(t){let e="",r=0;while(r<t.length){let n=t.indexOf("[",r);if(n===-1){e+=t.slice(r);break}let a=t.indexOf("]",n+1);if(a===-1||t[a+1]!=="("||a===n+1){e+=t.slice(r,n+1),r=n+1;continue}let l=t.indexOf(")",a+2);if(l===-1){e+=t.slice(r,n+1),r=n+1;continue}let i=t.slice(a+2,l).trimStart();if(!i.startsWith("http://")&&!i.startsWith("https://")){e+=t.slice(r,n+1),r=n+1;continue}e+=t.slice(r,n),e+=t.slice(n+1,a),r=l+1}return e}function tl(t){let e="",r=!1;for(let n of t)if(n===" "||n==="\t"||n===`
|
|
338
|
+
`||n==="\r"){if(!r)e+=" ";r=!0}else e+=n,r=!1;return e.trim()}function Nr(t){return new Set(String(t).toLowerCase().normalize("NFD").replaceAll(/[\u0300-\u036f]/g,"").split(/[^\w]+/).filter((e)=>e.length>1))}function jt(t,e){let r=Nr(t),n=Nr(e),a=new Set([...r,...n]).size;if(a===0)return 1;let l=0;for(let i of r)if(n.has(i))l++;return l/a}function Tr(t,e,{threshold:r=0.75,roundIndex:n=0,originalQuery:a=null}={}){let l=M(t).toLowerCase();if(e.has(l))return!0;if(a&&n>0&&l===M(a).toLowerCase())return!0;for(let i of e)if(jt(l,i)>=r)return!0;return!1}function el(t,e,r,n){let a=e.map((l)=>({queries:l.queries?.map((i)=>i.query||"")||[],learnings:l.learnings||[],gaps:l.gaps||[]}));return["You are evaluating the quality of an iterative research run.","Assess coverage across: official sources, limitations/risks, benchmarks/performance, production usage, and counter-evidence.","Score each dimension 0-10. Overall score 0-10.","Identify remaining knowledge gaps.","Propose targeted next actions (search queries or direct URL fetches) that would most improve the research.","Decide whether to continue or stop.","terminationReason must be one of: quality_threshold | max_rounds | no_novel_actions | insufficient_evidence.","",`Original research question: ${t}`,`Rounds completed: ${JSON.stringify(a,null,2)}`,`Accumulated learnings: ${JSON.stringify(r.slice(0,12),null,2)}`,`Known gaps: ${JSON.stringify(n.slice(0,8),null,2)}`,"","Respond ONLY with JSON wrapped in BEGIN_JSON / END_JSON markers:","BEGIN_JSON",JSON.stringify({score:7.5,coverage:{officialSources:8,limitations:5,benchmarks:7,productionUseCases:6,counterEvidence:4},knowledgeGaps:["specific gap or missing evidence"],shouldContinue:!0,terminationReason:"quality_threshold",nextActions:[{type:"search",query:"targeted search query"},{type:"fetchUrl",url:"https://example.com/primary-doc"}]},null,2),"END_JSON"].join(`
|
|
339
|
+
`)}function rl(t,e,r,n,a){let l=[],i=[{template:(f)=>`${f} official documentation`,label:"official docs"},{template:(f)=>`${f} GitHub issues discussions`,label:"community signals"},{template:(f)=>`${f} benchmarks performance comparison`,label:"benchmarks"},{template:(f)=>`${f} limitations risks caveats`,label:"limitations"},{template:(f)=>`${f} production deployment experience`,label:"production usage"},{template:(f)=>`${e} ${f} counter evidence`,label:"counter-evidence"}];for(let f=0;f<t.length&&l.length<n;f++){let g=t[f],o=i[f%i.length],p=o.template(g);if(!Tr(p,r,{roundIndex:a}))l.push({query:p,researchGoal:`Gap-driven: ${g} (${o.label})`})}return l}async function nl(t,e,r,n,a){try{let l=await x(el(t,e,r,n),{timeoutMs:120000}),i=lt(l,{}),f=typeof i.score==="number"?Math.min(10,Math.max(0,i.score)):a.length>0?a[a.length-1]:5,g=Array.isArray(i.knowledgeGaps)?i.knowledgeGaps.map((m)=>String(m)).filter(Boolean).slice(0,6):[],o=Array.isArray(i.nextActions)?i.nextActions.slice(0,5):[],p=typeof i.shouldContinue==="boolean"?i.shouldContinue:f<8,h=i.terminationReason||null;return{score:f,coverage:i.coverage||{},knowledgeGaps:g,shouldContinue:p,nextActions:o,terminationReason:h||(f>=8.5?"quality_threshold":null),evaluationError:""}}catch(l){return process.stderr.write(`[greedysearch] Quality evaluation failed: ${l.message}
|
|
340
|
+
`),{score:a.length>0?a[a.length-1]:5,coverage:{},knowledgeGaps:[],shouldContinue:!0,nextActions:[],terminationReason:null,evaluationError:l.message}}}function al(t){let e={};for(let r of Object.keys(t||{}).filter((n)=>!n.startsWith("_"))){let n=t?.[r];if(!n)continue;e[r]=n.error?{status:"error",error:String(n.error)}:{status:"ok",answer:O(n.answer||"",1400),sources:Array.isArray(n.sources)?n.sources.slice(0,5).map((a)=>({title:O(a.title||"",160),url:a.url||""})):[]}}return e}function ll(t,e,r=[],n=[],a=[]){let l=n.length>0?`
|
|
341
341
|
Known knowledge gaps to target:
|
|
342
342
|
${n.map((f)=>`- ${f}`).join(`
|
|
343
343
|
`)}`:"",i=a.length>0?`
|
|
@@ -347,45 +347,45 @@ ${a.map((f)=>`- ${f}`).join(`
|
|
|
347
347
|
Prior learnings to build on:
|
|
348
348
|
${r.map((f)=>`- ${f}`).join(`
|
|
349
349
|
`)}`:"",l,i,"","Respond ONLY with JSON wrapped in BEGIN_JSON / END_JSON markers:","BEGIN_JSON",JSON.stringify({actions:[{type:"search",query:"specific search query",researchGoal:"what this action should clarify"},{type:"fetchUrl",url:"https://example.com/docs/relevant-page",researchGoal:"extract specific information from this page"}]},null,2),"END_JSON"].join(`
|
|
350
|
-
`)}function
|
|
351
|
-
`)}function
|
|
352
|
-
`)}function
|
|
353
|
-
`)}function
|
|
354
|
-
`)}async function
|
|
355
|
-
`);
|
|
356
|
-
`)});let
|
|
357
|
-
`)}function
|
|
358
|
-
`):e}async function
|
|
359
|
-
`),"utf8"),
|
|
360
|
-
`),"utf8"),
|
|
361
|
-
`),"utf8"),
|
|
350
|
+
`)}function Lr(t){if(!t||typeof t!=="object")return null;let e=t.type,r=O(t.researchGoal||"",320);if(e==="search"){if(t.query==null)return null;let n=M(t.query);return n?{type:"search",query:n,researchGoal:r}:null}if(e==="fetchUrl"){if(t.url==null)return null;let n=A(t.url);return n?{type:"fetchUrl",url:n,researchGoal:r}:null}return null}async function il(t,{locale:e=null,short:r=!0,usedQueries:n,usedUrls:a,maxChars:l=8000}={}){if(t.type==="search"){let i=M(t.query).toLowerCase();n.add(i);try{let f=await El(t.query,{locale:e,short:r}),g=Tt(f,t.query);return{ok:!0,action:t,result:f,sources:g}}catch(f){return{ok:!1,action:t,error:f.message,sources:[]}}}if(t.type==="fetchUrl"){let i=A(t.url);if(a.has(i))return{ok:!1,action:t,error:`URL already fetched: ${i}`,sources:[]};try{let f=await fl(i,l);a.add(i);let g=gl(i),o={id:"",canonicalUrl:f.finalUrl||i,displayUrl:f.url||i,domain:g,title:f.title||i,engines:["fetch"],engineCount:1,perEngine:{},sourceType:Rt(g,f.title||"",f.finalUrl||i),isOfficial:!1,smartScore:0,fetch:{attempted:!0,ok:!f.error&&(f.contentChars||0)>100,status:f.status||null,finalUrl:f.finalUrl||i,content:f.content||"",contentChars:f.contentChars||0,snippet:f.snippet||"",error:f.error||""}};return{ok:!0,action:t,result:null,sources:[o],fetchResult:{id:o.id,url:i,finalUrl:f.finalUrl||i,title:f.title||"",content:f.content||"",contentChars:f.contentChars||0,snippet:f.snippet||"",status:f.status||null,error:f.error||"",source:f.source||"http",duration:f.duration||0}}}catch(f){return{ok:!1,action:t,error:f.message,sources:[]}}}return{ok:!1,action:t,error:`Unknown action type: ${t.type}`,sources:[]}}async function fl(t,e){return await ol(t,e)}async function ol(t,e=8000){let r=Date.now();try{let{parseGitHubUrl:n,fetchGitHubContent:a}=await Promise.resolve().then(() => (qt(),ut)),l=n(t);if(l&&(l.type==="root"||l.type==="tree"||l.type==="blob"&&!l.path?.includes("."))){let i=await a(t);if(i.ok){let{trimContentHeadTail:f}=await Promise.resolve().then(() => te),g=f(i.content,e);return{url:t,finalUrl:t,status:200,title:i.title,snippet:g.slice(0,320),content:g,contentChars:g.length,source:"github-api",duration:Date.now()-r}}}}catch{}try{let{fetchSourceHttp:n}=await Promise.resolve().then(() => (Cr(),jr)),{trimContentHeadTail:a}=await Promise.resolve().then(() => te),l=await n(t,{timeoutMs:1e4});if(l.ok){let i=a(l.markdown,e);return{url:t,finalUrl:l.finalUrl,status:l.status,title:l.title,snippet:l.excerpt,content:i,contentChars:i.length,source:"http",duration:Date.now()-r}}}catch{}return{url:t,title:"",content:"",contentChars:0,snippet:"",error:"HTTP fetch failed",source:"error",duration:Date.now()-r}}function gl(t){try{return new URL(t).hostname.toLowerCase().replace(/^www\./,"")}catch{return""}}async function pl(t,e){let r=[],{parseGitHubUrl:n}=await Promise.resolve().then(() => (qt(),ut));for(let a of t){if(a.type!=="fetchUrl"){r.push(a);continue}let l=n(a.url);if(!l||l.type!=="root"){r.push(a);continue}let{owner:i,repo:f}=l,g=`https://github.com/${i}/${f}`;if(e.has(g))continue;let o=[g],p=[`${g}/blob/main/CONTRIBUTING.md`,`${g}/blob/master/CONTRIBUTING.md`,`${g}/blob/main/CHANGELOG.md`,`${g}/blob/master/CHANGELOG.md`,`${g}/blob/main/docs/README.md`];for(let h of p){if(o.length>=3)break;if(!e.has(h))o.push(h)}for(let h of o)r.push({type:"fetchUrl",url:h,researchGoal:a.researchGoal||`Fetch GitHub content for ${i}/${f}`})}return r}function hl(t,e){let r=ct(t?.answer||"")||{},n=Array.isArray(r?.actions)?r.actions:[],a=[];for(let l of n){let i=Lr(l);if(i&&a.length<e)a.push(i)}return a}function cl(t){return(t||[]).map((e)=>({type:"search",query:typeof e==="string"?e:e.query,researchGoal:typeof e==="string"?"":e.researchGoal||""})).filter((e)=>e.query)}function kt(t){return A(t?.finalUrl||t?.canonicalUrl||t?.url||"")||t?.id||""}function wl(t,e,r,n=new Set){let a=(e||[]).filter((i)=>i.status!=="closed").slice(0,12).map((i)=>({id:i.id,question:i.question})),l=(r||[]).filter((i)=>i?.content||i?.snippet).filter((i)=>!n.has(kt(i))).slice(0,6).map((i,f)=>({id:i.id||`F${f+1}`,title:i.title||"",url:i.finalUrl||i.url||i.canonicalUrl||"",content:O(i.content||i.snippet||"",5000)}));return["You are doing goal-based evidence extraction for an iterative research run.","For each source, extract only information that helps answer the open questions.","Use original wording/details where useful. Do not invent answers; leave questions open if evidence is insufficient.","If a source answers one or more tracked questions, identify those question IDs explicitly.","Also propose genuinely new sub-questions discovered from the evidence.","",`Original research question: ${t}`,`Open question ledger: ${JSON.stringify(a,null,2)}`,`Fetched sources: ${JSON.stringify(l,null,2)}`,"","Respond ONLY with JSON wrapped in BEGIN_JSON / END_JSON markers:","BEGIN_JSON",JSON.stringify({extractions:[{sourceId:"S1",url:"https://example.com/source",rational:"why this source matters for the goal",evidence:"specific quoted/paraphrased evidence with numbers, dates, caveats",summary:"concise contribution to the research question",answers:[{id:"Q1",evidence:"brief evidence that closes the question"}],newQuestions:["new sub-question raised by this source"]}]},null,2),"END_JSON"].join(`
|
|
351
|
+
`)}function $l(t,e){let r=Array.isArray(t?.extractions)?t.extractions:[],n=new Map,a=new Map;for(let l of e||[]){if(l?.id)a.set(String(l.id),l);let i=kt(l);if(i)n.set(i,l)}return r.map((l)=>{let i=a.get(String(l?.sourceId||""))||n.get(A(l?.url||"")||""),f=String(l?.sourceId||i?.id||""),g=A(l?.url||i?.finalUrl||i?.url||""),o=Array.isArray(l?.answers)?l.answers.map((p)=>({id:String(p?.id||""),evidence:O(p?.evidence||"",500),sourceIds:[f].filter(Boolean)})).filter((p)=>p.id):[];return{sourceId:f,url:g,title:i?.title||l?.title||"",rational:O(l?.rational||"",700),evidence:O(l?.evidence||"",1600),summary:O(l?.summary||"",700),answers:o,newQuestions:Z(l?.newQuestions||[],6)}}).filter((l)=>l.sourceId||l.url||l.summary||l.evidence)}async function ml({query:t,questions:e,fetchedSources:r,extractedSourceKeys:n}){let a=(r||[]).filter((l)=>(l?.content||l?.snippet)&&!n.has(kt(l)));if(a.length===0)return{evidence:[],error:""};try{let l=await x(wl(t,e,a,n),{timeoutMs:120000}),i=lt(l,{extractions:[]}),f=$l(i,a);for(let g of a){let o=kt(g);if(o)n.add(o)}return{evidence:f,error:""}}catch(l){return{evidence:[],error:l.message||String(l)}}}function bl(t,e,r,n,a=[],l=[]){let i=n.filter((f)=>f?.content||f?.snippet).slice(0,10).map((f,g)=>({id:`F${g+1}`,title:f.title||"",url:f.finalUrl||f.url||"",snippet:O(f.content||f.snippet||"",3000)}));return["You are extracting compact research state from live multi-engine search results.","Create dense, non-overlapping learnings with exact names, numbers, dates, limitations, and caveats where available.","Also propose follow-up search queries that would most improve confidence or fill gaps.","",`Original research question: ${t}`,`Round queries: ${JSON.stringify(e,null,2)}`,`Question ledger: ${JSON.stringify(a,null,2)}`,`Extracted source evidence: ${JSON.stringify(l.slice(-12),null,2)}`,`Engine summaries: ${JSON.stringify(r,null,2)}`,`Fetched source snippets: ${JSON.stringify(i,null,2)}`,"","Respond ONLY with JSON wrapped in BEGIN_JSON / END_JSON markers:","BEGIN_JSON",JSON.stringify({learnings:["concise, information-dense learning"],answeredQuestions:[{id:"Q1",evidence:"brief evidence that closes this question",sourceIds:["S1"]}],newQuestions:["new sub-question discovered from the evidence"],followUpQueries:["specific next search query"],gaps:["important uncertainty or missing evidence"]},null,2),"END_JSON"].join(`
|
|
352
|
+
`)}function sl(t,e,r,n=[],a=[]){let l=e.flatMap((g)=>g.learnings||[]),i=e.flatMap((g)=>g.gaps||[]),f=r.slice(0,12).map((g)=>({id:g.id,title:g.title,domain:g.domain,url:g.canonicalUrl,type:g.sourceType,engines:g.engines,fetch:g.fetch?.attempted?{ok:g.fetch.ok,snippet:O(g.fetch.snippet||"",1200),publishedTime:g.fetch.publishedTime||""}:void 0}));return["You are writing the final research report for an iterative deep-research run.","Produce a thorough markdown report organized into clear sections.","","Use the learnings and source registry below. Every substantive claim MUST be backed by an [S1] citation.",'Where engines disagree, surface the conflicting claims explicitly in the "differences" array.','Include a "Key Claims" structure that maps each distinct claim to its supporting source IDs.',"","Report structure:","1. ## Summary — A 2-4 sentence executive summary of findings","2. ## Key Findings — The main findings, organized by theme or question, each with inline citations","3. ## Areas of Disagreement — Where engines or sources conflict (if any)","4. ## Limitations & Caveats — Important qualifiers, gaps, or uncertainties","",`Original research question: ${t}`,`Learnings: ${JSON.stringify(l,null,2)}`,`Known gaps/caveats: ${JSON.stringify(i,null,2)}`,`Question ledger: ${JSON.stringify(n,null,2)}`,`Goal-based extracted evidence: ${JSON.stringify(a.slice(-20),null,2)}`,`Source registry: ${JSON.stringify(f,null,2)}`,"","Respond ONLY with JSON wrapped in BEGIN_JSON / END_JSON markers:","BEGIN_JSON",JSON.stringify({answer:"markdown report with sections and inline [S1] citations",agreement:{level:"high|medium|low|mixed|conflicting",summary:"one-sentence confidence summary"},differences:["notable disagreement or conflict between sources"],caveats:["important caveat or qualification"],claims:[{claim:"specific factual statement from the research",support:"strong|moderate|weak|conflicting",sourceIds:["S1","S2"]}],recommendedSources:["S1","S2"]},null,2),"END_JSON"].join(`
|
|
353
|
+
`)}function _l(t,e=[],r=[],n=[]){let a=e.slice(0,12).map((g)=>({id:g.id,title:g.title,domain:g.domain,url:g.canonicalUrl,type:g.sourceType,engines:g.engines})),l=n.slice(-20),i=new Set;for(let g of l)for(let o of g.answers||[])if(o?.id)i.add(o.id);let f=(r||[]).filter((g)=>g.status!=="closed").map((g)=>({id:g.id,question:g.question}));return["You are writing the final research report from goal-based extracted evidence.","Per-round learnings were not produced, but the per-source evidence extraction step succeeded.","Synthesize a thorough markdown report using ONLY the evidence below. Every substantive claim MUST be backed by an [S1] citation.","","Report structure:","1. ## Summary — A 2-4 sentence executive summary of findings","2. ## Key Findings — The main findings, organized by theme or question, each with inline citations","3. ## Limitations & Caveats — Important qualifiers, gaps, or uncertainties","",`Original research question: ${t}`,`Per-source extracted evidence: ${JSON.stringify(l,null,2)}`,`Source registry: ${JSON.stringify(a,null,2)}`,`Questions already answered by the evidence: ${JSON.stringify(Array.from(i))}`,`Questions still open after this evidence: ${JSON.stringify(f)}`,"","Respond ONLY with JSON wrapped in BEGIN_JSON / END_JSON markers:","BEGIN_JSON",JSON.stringify({answer:"markdown report with sections and inline [S1] citations",agreement:{level:"high|medium|low|mixed|conflicting",summary:"one-sentence confidence summary"},differences:["notable disagreement or conflict between sources"],caveats:["important caveat or qualification"],claims:[{claim:"specific factual statement supported by the evidence",support:"strong|moderate|weak|conflicting",sourceIds:["S1","S2"]}],recommendedSources:["S1","S2"]},null,2),"END_JSON"].join(`
|
|
354
|
+
`)}async function El(t,{locale:e=null,short:r=!0}={}){let n=[Xa,"all","--inline","--stdin","--fast"];if(!r)n.push("--full");if(e)n.push("--locale",e);return new Promise((a,l)=>{let i=Ka(wt(),n,{stdio:["pipe","pipe","pipe"],env:{...process.env,GREEDY_SEARCH_RESEARCH_CHILD:"1"}});i.stdin.write(t),i.stdin.end();let f="",g="",o="",p=!1;i.stdout.on("data",(m)=>f+=m),i.stderr.on("data",(m)=>{g+=m,o+=m.toString();let w=o.split(`
|
|
355
|
+
`);o=w.pop()||"";for(let s of w)if(Dl(s))process.stderr.write(`${s}
|
|
356
|
+
`)});let h=setTimeout(()=>{p=!0,i.kill()},140000);i.on("close",(m)=>{clearTimeout(h);let w=p?`research child search timed out for: ${t}`:g.trim()||`search child exited with code ${m}`;if(p||m!==0){let b=Ce(f,{code:m,error:w,timedOut:p});if(b){a(b);return}l(Error(w));return}let s=Ft(f);if(s)a(s);else l(Error(`Invalid JSON from research child: ${f.slice(0,200)}`))})})}function yl(t){let e=new Map;for(let r of t.flat()){let n=A(r.canonicalUrl||r.url);if(!n)continue;let a=e.get(n);if(!a){e.set(n,{...r,canonicalUrl:n});continue}a.engines=[...new Set([...a.engines||[],...r.engines||[]])],a.engineCount=a.engines.length,a.smartScore=Math.max(a.smartScore||0,r.smartScore||0)}return Array.from(e.values()).sort((r,n)=>{let a=X(n)-X(r);if(a!==0)return a;return(r.domain||"").localeCompare(n.domain||"")}).slice(0,12).map((r,n)=>({...r,id:`S${n+1}`}))}function Dl(t){return/^PROGRESS:/.test(t)||/^\[greedysearch\]/.test(t)||/^\[(bing|perplexity|google|gemini|chatgpt|logically|semantic-scholar)\]/.test(t)||/^GreedySearch Chrome/.test(t)||/^Launching GreedySearch Chrome/.test(t)||/^Headless mode/.test(t)||/^Ready\.?$/.test(t)}function lt(t,e={}){return ct(t?.answer||"")||e}function vl(t,e){if(!t||!Array.isArray(e))return{cited:[],missing:[],unfetched:[],ok:!0};let r=/\b[SF](\d+)\b/g,n=new Set,a;while((a=r.exec(t))!==null)n.add(`S${a[1]}`),n.add(`F${a[1]}`);let l=new Map;for(let o of e){let p=o?.id;if(p)l.set(p,o)}let i=Array.from(n),f=[],g=[];for(let o of i){let p=l.get(o);if(!p){let h=o.match(/^(S|F)(\d+)$/);if(h){let m=parseInt(h[2],10)-1;if(m>=0&&m<e.length){let w=e[m];if(w){if(!(w.fetch?.ok||w.content&&w.content.length>100||w.contentChars&&w.contentChars>100))g.push(o);continue}}}f.push(o)}else if(!(p.fetch?.ok||p.content&&p.content.length>100||p.contentChars&&p.contentChars>100))g.push(o)}return{cited:i,missing:f,unfetched:g,ok:f.length===0}}function Or({sources:t=[],fetchedSources:e=[],synthesis:r={},citationAudit:n=null,gaps:a=[],questions:l=[],rounds:i=[],qualityScore:f=0,qualityThreshold:g=8.5,maxSources:o=8,requireCitations:p=!0,requireQuestions:h=!0}={}){let m=e.filter((N)=>N?.fetch?.ok||(N?.contentChars||0)>100||String(N?.content||"").length>100),w=t.filter((N)=>["official-docs","repo","maintainer-blog","academic"].includes(String(N?.sourceType||""))),s=Array.isArray(r?.claims)?r.claims:[],b=n?n.cited?.length||0:0,c=fe(l),E=(l||[]).filter((N)=>!N.createdRound||N.reason==="Original research question"),C=fe(E),_=Math.min(4,Math.max(2,Number(o)||8)),y={roundsRun:i.length>=1,fetchedSources:m.length>=_,primarySources:w.length>=1,qualityScore:f>=Math.min(g,8),claimsExtracted:!p||s.length>0,citationsPresent:!p||b>0,citationsValid:!p||n?.ok===!0,unfetchedCitations:!p||(n?.unfetched||[]).length===0,requiredQuestionsClosed:!h||C.open===0};return{floorMet:Object.values(y).every(Boolean),checks:y,metrics:{fetchedOk:m.length,primarySources:w.length,claims:s.length,cited:b,gaps:a.length,openQuestions:c.open,closedQuestions:c.closed,totalQuestions:c.total,openRequiredQuestions:C.open,closedRequiredQuestions:C.closed,totalRequiredQuestions:C.total,qualityScore:f,minFetched:_}}}function Pr(t,e){let r=new Map;for(let n of e||[]){let a=A(n?.canonicalUrl||n?.finalUrl||n?.url);if(a&&n?.id)r.set(a,n.id)}return(t||[]).map((n,a)=>{let l=A(n?.finalUrl||n?.canonicalUrl||n?.url);return{...n,id:n?.id||r.get(l)||`F${a+1}`}})}function kl(t){return[{id:"Q1",question:O(M(t),320),status:"open",reason:"Original research question",evidence:[],sourceIds:[]}]}function jl(t){let e=0;for(let r of t||[]){let n=Number.parseInt(String(r.id||"").replace(/^Q/i,""),10);if(Number.isFinite(n))e=Math.max(e,n)}return`Q${e+1}`}function Ar(t,e){let r=M(e).toLowerCase();return(t||[]).find((n)=>n.question?.toLowerCase()===r||jt(n.question||"",r)>=0.82)}function ae(t,e,{reason:r="",round:n=null}={}){let a=O(M(e),320);if(!a)return null;let l=Ar(t,a);if(l)return l;let i={id:jl(t),question:a,status:"open",reason:O(r,240),createdRound:n,evidence:[],sourceIds:[]};return t.push(i),i}function vt(t,e,{evidence:r="",sourceIds:n=[],round:a=null}={}){let l=t.find((i)=>i.id===e)||Ar(t,e);if(!l)return null;if(l.status="closed",l.closedRound=l.closedRound||a,r)l.evidence=Z([...l.evidence||[],r],4);if(Array.isArray(n))l.sourceIds=Z([...l.sourceIds||[],...n],8);return l}function fe(t){let e=t.length,r=t.filter((n)=>n.status==="closed").length;return{total:e,closed:r,open:Math.max(0,e-r)}}function Dt(t,{roundNumber:e,actions:r=[],learningPayload:n={}}={}){for(let g of r){let o=g?.action||g,p=o?.researchGoal&&o.researchGoal!=="Original user query"?o.researchGoal:o?.query||o?.url||"";if(p)ae(t,p,{reason:"Planned research action",round:e})}let a=5,l=t.filter((g)=>g.status==="open"&&g.reason==="Discovered gap/follow-up");if(l.length>a){let g=l.sort((o,p)=>(o.createdRound||0)-(p.createdRound||0)).slice(0,l.length-a);for(let o of g)o.status="resolved",o.closedRound=e,o.evidence=Z([...o.evidence||[],"Auto-resolved to cap open-question ledger"],4)}let i=Array.isArray(n.answeredQuestions)?n.answeredQuestions:[];for(let g of i){if(typeof g==="string"){vt(t,g,{round:e});continue}let o=g?.id||g?.question;if(!o&&g?.question){let p=ae(t,g.question,{reason:"Answered during learning extraction",round:e});if(p)vt(t,p.id,{round:e});continue}vt(t,o,{evidence:g?.evidence||g?.answer||"",sourceIds:Array.isArray(g?.sourceIds)?g.sourceIds:[],round:e})}let f=Array.isArray(n.newQuestions)?n.newQuestions:[];for(let g of f)ae(t,g,{reason:"Discovered gap/follow-up",round:e});return t}function Cl(t,e){if(!Array.isArray(t)||t.length===0)return[];let r=["arxiv.org","semanticscholar.org","doi.org"],n=new Set,a=[];for(let l of t){let i=l?.canonicalUrl||l?.finalUrl||l?.url||"";if(!i)continue;let f="";try{f=new URL(i).hostname.toLowerCase().replace(/^www\./,"")}catch{continue}if(!r.some((o)=>f===o||f.endsWith(`.${o}`)))continue;if(e.has(i)||n.has(i))continue;n.add(i);let g=i.includes("/pdf/")?i.replace(/\/pdf\//,"/html/").replace(/\.pdf$/i,""):i;a.push({url:g,label:l?.title||l?.id||f})}return a.slice(0,2)}function Nl(t,e,r){if(!e?.answer||r?.ok!==!0)return t;let n=Array.isArray(e.claims)?e.claims:[],a=Array.isArray(r.cited)?r.cited:[];if(n.length===0||a.length===0)return t;for(let l of t){if(l.status==="closed")continue;let i=null,f=0;for(let g of n){let o=jt(l.question||"",g.claim||"");if(o>f)f=o,i=g}if(l.id==="Q1"||f>=0.18)vt(t,l.id,{evidence:i?.claim||"Answered in final cited synthesis",sourceIds:Array.isArray(i?.sourceIds)?i.sourceIds:a.slice(0,4)})}return t}function Ol(t){if(!t.length)return"No tracked questions.";return t.map((e)=>{let r=e.sourceIds?.length?` (${e.sourceIds.join(", ")})`:"";return`- [${e.status==="closed"?"x":" "}] ${e.id}: ${e.question}${r}`}).join(`
|
|
357
|
+
`)}function le(t,e="None recorded."){let r=Z(t);return r.length?r.map((n)=>`- ${n}`).join(`
|
|
358
|
+
`):e}async function Pl({query:t,rounds:e,sources:r,fetchedSources:n,evidenceItems:a=[],synthesis:l,citationAudit:i,floor:f,manifest:g,allGaps:o=[],questions:p=[],outDir:h=null}){let m=new Date().toISOString().replaceAll(/[:.]/g,"-").slice(0,19),w=h||j(xa,`${m}_${Qa(t)}`),s=j(w,"reports"),b=j(w,"sources"),c=j(w,"data");ee(s,{recursive:!0}),ee(b,{recursive:!0}),ee(c,{recursive:!0});let E=await ie(n,b),C=Z([...o,...e.flatMap((_)=>_.gaps||[])]);return L(j(w,"STATUS.md"),[f.floorMet?"STATUS: DONE":"STATUS: PARTIAL","",`Query: ${t}`,`Stop reason: ${g.terminationReason||"max_rounds"}`,"","## Deterministic floor checks",...Object.entries(f.checks).map(([_,y])=>`- [${y?"x":" "}] ${_}`),"","## Questions",Ol(p),"","## Open gaps",le(C),""].join(`
|
|
359
|
+
`),"utf8"),L(j(w,"OUTLINE.md"),["# Research bundle outline","","- `reports/SUMMARY.md` — final cited report","- `reports/CLAIMS.md` — extracted claims with support/source IDs","- `reports/EVIDENCE.md` — goal-based source evidence","- `reports/GAPS.md` — remaining caveats and uncertainties","- `sources/` — fetched source markdown files","- `data/manifest.json` — machine-readable run metadata","- `data/rounds.json` — per-round actions/learnings/gaps","- `data/sources.json` — ranked source registry","- `data/questions.json` — open/closed question ledger",""].join(`
|
|
360
|
+
`),"utf8"),L(j(s,"SUMMARY.md"),String(l.answer||""),"utf8"),L(j(s,"CLAIMS.md"),["# Key claims","",...Array.isArray(l.claims)&&l.claims.length?l.claims.map((_)=>{let y=Array.isArray(_.sourceIds)?_.sourceIds.join(", "):"";return`- ${_.claim||""} (${_.support||"support unknown"}${y?`; ${y}`:""})`}):["No structured claims were extracted."],""].join(`
|
|
361
|
+
`),"utf8"),L(j(s,"EVIDENCE.md"),["# Extracted evidence","",...a.length?a.map((_)=>[`## ${_.sourceId||_.url||"Source"}`,_.url?`<${_.url}>`:"",_.rational?`**Rational:** ${_.rational}`:"",_.evidence?`**Evidence:** ${_.evidence}`:"",_.summary?`**Summary:** ${_.summary}`:"",""].filter(Boolean).join(`
|
|
362
362
|
`)):["No goal-based evidence was extracted."],""].join(`
|
|
363
|
-
`),"utf8"),
|
|
364
|
-
`),"utf8"),
|
|
365
|
-
`),"utf8"),{dir:
|
|
366
|
-
`);for(let
|
|
367
|
-
`),!
|
|
368
|
-
`);let
|
|
369
|
-
`);return
|
|
370
|
-
`);return
|
|
371
|
-
`)}let
|
|
372
|
-
`),process.stderr.write(`[greedysearch] Action ${$+1}/${
|
|
373
|
-
`);let
|
|
374
|
-
`)}let $e=
|
|
375
|
-
`);let $=await
|
|
376
|
-
`);let u=await
|
|
377
|
-
`);
|
|
378
|
-
`);let
|
|
379
|
-
`)}let
|
|
380
|
-
`);let
|
|
381
|
-
`),
|
|
382
|
-
`);break}let
|
|
383
|
-
`)}
|
|
384
|
-
`);let
|
|
385
|
-
`):"Research completed, but no structured learnings were extracted.",agreement:{level:"mixed",summary:"Research synthesis fallback."},differences:[],caveats:[],claims:[],recommendedSources:b.slice(0,4).map((
|
|
386
|
-
`),
|
|
387
|
-
`);try{let
|
|
388
|
-
`),
|
|
389
|
-
`);let it=
|
|
390
|
-
`);try{tt=await
|
|
391
|
-
`),{query:t,_research:{mode:"iterative",breadth:
|
|
363
|
+
`),"utf8"),L(j(s,"GAPS.md"),["# Gaps and caveats","","## Caveats",le(l.caveats||[]),"","## Research gaps",le(C),""].join(`
|
|
364
|
+
`),"utf8"),L(j(c,"manifest.json"),JSON.stringify({...g,floor:f,citationAudit:i},null,2),"utf8"),L(j(c,"rounds.json"),JSON.stringify(e,null,2),"utf8"),L(j(c,"sources.json"),JSON.stringify(r,null,2),"utf8"),L(j(c,"questions.json"),JSON.stringify(p,null,2),"utf8"),L(j(c,"evidence.json"),JSON.stringify(a,null,2),"utf8"),L(j(b,"index.md"),["# Source index","",...E.map((_)=>{let y=_.title||_.url,N=_.finalUrl||_.url,W=_.contentPath?` — ${_.contentPath}`:"";return`- ${_.id||"?"}: [${y}](${N})${W}`}),""].join(`
|
|
365
|
+
`),"utf8"),{dir:w,statusPath:j(w,"STATUS.md"),summaryPath:j(s,"SUMMARY.md"),manifestPath:j(c,"manifest.json"),sourceCount:E.length,sourceFiles:E}}async function Gf({query:t,breadth:e=3,iterations:r=2,maxSources:n,locale:a=null,short:l=!1,qualityThreshold:i=8.5,writeBundle:f=process.env.GREEDY_RESEARCH_BUNDLE!=="0",researchOutDir:g=null}={}){let o=za({breadth:e,iterations:r,maxSources:n}),p=[],h=[],m=[],w=kl(t),s=null,b=[],c=[],E=[],C=new Set,_=new Set,y=new Set,N=[],W="max_rounds",Mr=new Date().toISOString(),Wr=Date.now(),oe=0,ge=0,pe=0,he=[];process.stderr.write(`[greedysearch] Research mode: breadth ${o.breadth}, iterations ${o.iterations}, qualityThreshold ${i}, engines ${Ht.join(",")}, synthesizer gemini
|
|
366
|
+
`);for(let D=0;D<o.iterations;D++){let k=D+1,H=Math.max(1,Math.ceil(o.breadth/2**D));if(process.stderr.write(`PROGRESS:research:round-${k}:planning
|
|
367
|
+
`),!s)try{let $=await x(ll(t,H,h,m,[...y]),{timeoutMs:120000}),v=hl($,H);if(D===0)v.unshift({type:"search",query:t,researchGoal:"Original user query"});v=await pl(v,y),s=v}catch($){process.stderr.write(`[greedysearch] Action planning failed, using fallback queries: ${$.message}
|
|
368
|
+
`);let v=ua(null,t,H,{includeOriginal:D===0,exclude:_});s=cl(v)}let S=(s||[]).filter(($)=>{if($.type==="search"){let v=!Tr($.query,_,{roundIndex:D,originalQuery:t});if(!v)process.stderr.write(`[greedysearch] Novelty gate rejected search: ${$.query}
|
|
369
|
+
`);return v}if($.type==="fetchUrl"){let v=!y.has($.url);if(!v)process.stderr.write(`[greedysearch] Novelty gate rejected fetch: ${$.url}
|
|
370
|
+
`);return v}return!1}).slice(0,H),we=Cl(b,y);if(!S.some(($)=>$.type==="fetchUrl")&&we.length>0){let $=we[0];S.push({type:"fetchUrl",url:$.url,researchGoal:`Direct fetch of known academic source: ${$.label||$.url}`}),process.stderr.write(`[greedysearch] Forced fetchUrl for academic source: ${$.url}
|
|
371
|
+
`)}let z=[];for(let $=0;$<S.length;$++){let v=S[$];process.stderr.write(`PROGRESS:research:round-${k}:action-${$+1}/${S.length}
|
|
372
|
+
`),process.stderr.write(`[greedysearch] Action ${$+1}/${S.length} [${v.type}]: ${(v.query||v.url).slice(0,80)}
|
|
373
|
+
`);let I=await il(v,{locale:a,short:l,usedQueries:_,usedUrls:y,maxChars:8000});if(z.push(I),oe++,v.type==="search")ge++;if(v.type==="fetchUrl")pe++;if(!I.ok)he.push({round:k,type:v.type,target:v.query||v.url,error:I.error}),process.stderr.write(`[greedysearch] Action failed: ${I.error}
|
|
374
|
+
`)}let $e=z.filter(($)=>$.action.type==="search"),me=z.filter(($)=>$.action.type==="fetchUrl");Dt(w,{roundNumber:k,actions:z}),b=yl([b,$e.flatMap(($)=>$.sources||[]),me.flatMap(($)=>$.sources||[])]);for(let $ of me)if($.fetchResult)c.push($.fetchResult);c=Rr(c);let Ct=Math.max(0,o.maxSources-c.filter(($)=>$?.content||$?.contentChars>100).length);if(Ct>0&&b.length>0){process.stderr.write(`PROGRESS:research:round-${k}:fetching
|
|
375
|
+
`);let $=await Sa(b,Math.min(Ct,b.length),8000,Math.min(3,Ct||1));c=Rr([...c,...$]),b=De(b,c)}c=Pr(c,b),process.stderr.write(`PROGRESS:research:round-${k}:evidence
|
|
376
|
+
`);let u=await ml({query:t,questions:w,fetchedSources:c,extractedSourceKeys:C});if(u.error)process.stderr.write(`[greedysearch] Evidence extraction failed: ${u.error}
|
|
377
|
+
`);E=[...E,...u.evidence];for(let $ of u.evidence)Dt(w,{roundNumber:k,learningPayload:{answeredQuestions:$.answers||[],newQuestions:$.newQuestions||[]}});let Ir=z.map(($)=>({query:$.action.query||$.action.url||"",researchGoal:$.action.researchGoal||""}));process.stderr.write(`PROGRESS:research:round-${k}:learning
|
|
378
|
+
`);let F={learnings:[],followUpQueries:[],gaps:[]},be="";try{let $=await x(bl(t,Ir,$e.map((v)=>({query:v.action.query,researchGoal:v.action.researchGoal,error:v.error||"",engines:al(v.result)})),c,w,E),{timeoutMs:120000});F={...F,...lt($,F)}}catch($){be=$.message,process.stderr.write(`[greedysearch] Learning extraction failed: ${$.message}
|
|
379
|
+
`)}let se=Array.isArray(F.learnings)?F.learnings.map(($)=>String($)).filter(Boolean).slice(0,8):[],Nt=Array.isArray(F.gaps)?F.gaps.map(($)=>String($)).filter(Boolean).slice(0,6):[];h=Z([...h,...se]),m=Z([...m,...Nt]),Dt(w,{roundNumber:k,actions:[],learningPayload:F,gaps:Nt}),p.push({round:k,actions:z.map(($)=>({type:$.action.type,query:$.action.query||"",url:$.action.url||"",researchGoal:$.action.researchGoal||"",error:$.error||"",sourceCount:$.sources?.length||0})),learnings:se,gaps:Nt,evidence:u.evidence,evidenceError:u.error,learningError:be}),process.stderr.write(`PROGRESS:research:round-${k}:evaluating
|
|
380
|
+
`);let T=await nl(t,p,h,m,N);N.push(T.score),m=Z([...m,...T.knowledgeGaps||[]]),Dt(w,{roundNumber:k,gaps:T.knowledgeGaps||[]});let _e=Or({sources:b,fetchedSources:c,gaps:m,questions:w,rounds:p,qualityScore:T.score,qualityThreshold:i,maxSources:o.maxSources,requireCitations:!1,requireQuestions:!1});if(process.stderr.write(`[greedysearch] Quality score round ${k}: ${T.score.toFixed(1)} (shouldContinue: ${T.shouldContinue}, floor: ${_e.floorMet})
|
|
381
|
+
`),T.score>=i&&_e.floorMet&&(!T.shouldContinue||T.terminationReason==="quality_threshold")){W=T.terminationReason||"quality_threshold",process.stderr.write(`[greedysearch] Research floor reached (score: ${T.score.toFixed(1)}). Terminating early.
|
|
382
|
+
`);break}let U=Math.max(1,Math.ceil(H/2)),G=(F.followUpQueries||[]).map(($)=>({type:"search",query:M(String($)),researchGoal:"Follow-up from learning extraction"})).filter(($)=>$.query&&$.query.toLowerCase()!==t.toLowerCase()).slice(0,U);if(G.length<U&&T.nextActions.length>0){let $=T.nextActions.map((I)=>Lr(I)).filter(Boolean);G=[...G,...$].slice(0,U)}if(G.length<U&&m.length>0){let $=rl(m,t,_,U-G.length,D+1),v=$.map((I)=>({type:"search",query:I.query,researchGoal:I.researchGoal}));if(G=[...G,...v].slice(0,U),$.length>0)process.stderr.write(`[greedysearch] Generated ${$.length} gap-driven fallback actions.
|
|
383
|
+
`)}s=G.length>=U?G:null}process.stderr.write(`PROGRESS:research:final-report
|
|
384
|
+
`);let P={answer:h.length?h.map((D)=>`- ${D}`).join(`
|
|
385
|
+
`):"Research completed, but no structured learnings were extracted.",agreement:{level:"mixed",summary:"Research synthesis fallback."},differences:[],caveats:[],claims:[],recommendedSources:b.slice(0,4).map((D)=>D.id),synthesized:!1};try{let D=await x(sl(t,p,b,w,E),{timeoutMs:180000}),k=lt(D,{}),H=Array.isArray(k?.claims)&&k.claims.length>0;P={...P,...k,rawAnswer:D.answer||"",geminiSources:D.sources||[],synthesized:H}}catch(D){process.stderr.write(`[greedysearch] Final report failed: ${D.message}
|
|
386
|
+
`),P.error=D.message}if(!(P.synthesized===!0&&Array.isArray(P.claims)&&P.claims.length>0)&&E.length>0){process.stderr.write(`[greedysearch] Falling back to evidence-based synthesis (no per-round learnings).
|
|
387
|
+
`);try{let D=_l(t,b,w,E),k=await x(D,{timeoutMs:180000}),H=lt(k,{});P={...P,...H,rawAnswer:k.answer||P.answer||"",geminiSources:k.sources||P.geminiSources||[],synthesized:!0,synthesisMode:"evidence_fallback"}}catch(D){process.stderr.write(`[greedysearch] Evidence-based synthesis failed: ${D.message}
|
|
388
|
+
`),P.evidenceFallbackError=D.message}}let Hr=new Date().toISOString(),Fr=Date.now()-Wr,Gr=N.at(-1)||0;c=Pr(c,b),process.stderr.write(`PROGRESS:research:audit-citations
|
|
389
|
+
`);let it=vl(P.answer||"",b);Nl(w,P,it);let Q=Or({sources:b,fetchedSources:c,synthesis:P,citationAudit:it,gaps:m,questions:w,rounds:p,qualityScore:Gr,qualityThreshold:i,maxSources:o.maxSources});if(Q.floorMet&&W==="max_rounds")W="done_floor_met";else if(!Q.floorMet&&W==="quality_threshold")W="max_rounds_floor_unmet";let ce={startedAt:Mr,finishedAt:Hr,durationMs:Fr,engines:Ht,synthesizer:"gemini",rounds:p.length,actionsRun:oe,searches:ge,fetches:pe,sourcesFetched:c.filter((D)=>D?.contentChars>100).length,engineFailures:he,terminationReason:W,floorMet:Q.floorMet},tt=null,ft;if(f){process.stderr.write(`PROGRESS:research:bundle
|
|
390
|
+
`);try{tt=await Pl({query:t,rounds:p,sources:b,fetchedSources:c,evidenceItems:E,synthesis:P,citationAudit:it,floor:Q,manifest:ce,allGaps:m,questions:w,outDir:g}),ft=tt.sourceFiles,delete tt.sourceFiles}catch(D){tt={error:D.message||String(D)},ft=await ie(c)}}else ft=await ie(c);return process.stderr.write(`PROGRESS:research:done
|
|
391
|
+
`),{query:t,_research:{mode:"iterative",breadth:o.breadth,iterations:o.iterations,maxSources:o.maxSources,rounds:p,learnings:h,gaps:m,evidence:E,questions:w,questionProgress:fe(w),qualityHistory:N,terminationReason:W,qualityThreshold:i,floor:Q,bundle:tt,manifest:ce},_citationAudit:it,_sources:b,_fetchedSources:ft,_synthesis:P,_confidence:{sourcesCount:b.length,fetchedSourceSuccessRate:c.length>0?Number((c.filter((D)=>D.contentChars>100).length/c.length).toFixed(2)):0,agreementLevel:P.agreement?.level||"mixed",floorMet:Q.floorMet}}}function Rr(t){let e=new Map;for(let n of t){let a=n?.id||A(n?.finalUrl||n?.url||"");if(!a)continue;let l=e.get(a);if(!l||(n.contentChars||0)>(l.contentChars||0))e.set(a,n)}let r=[];for(let n of e.values()){let a=String(n.content||n.snippet||""),l=r.findIndex((i)=>{let f=String(i.content||i.snippet||"");if(a.length<400||f.length<400)return!1;return jt(a.slice(0,4000),f.slice(0,4000))>=0.9});if(l===-1){r.push(n);continue}if((n.contentChars||0)>(r[l].contentChars||0))r[l]=n}return r}export{Lr as validateAction,Dt as updateQuestionLedger,Nr as tokenSet,Gf as runResearchMode,cl as queriesToActions,hl as parseActionPlan,ua as normalizeResearchQueries,jt as jaccardSimilarity,Tr as isDuplicateQuery,kl as createQuestionLedger,Or as computeResearchFloor,za as clampResearchOptions,rl as buildFallbackQueriesFromGaps,vl as auditCitations};
|