@duckmind/dm-windows-x64 0.59.3 → 0.60.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dm.exe +0 -0
- package/extensions/.dm-extensions.json +11 -7
- package/extensions/dm-cliproxy/index.js +1 -1
- package/extensions/dm-cliproxy/src/apply.js +1 -1
- package/extensions/dm-cliproxy/src/commands.js +1 -1
- package/extensions/dm-cliproxy/src/fetch-models.js +1 -1
- package/extensions/dm-cliproxy/src/ui-hub/hub.js +1 -1
- package/extensions/dm-cliproxy/src/ui-hub/index.js +1 -1
- package/extensions/dm-cua/bin/browser-cua.mjs +4 -4
- package/extensions/dm-cua/index.js +3 -3
- package/extensions/dm-cua/src/browser-cua-lib.mjs +3 -3
- package/extensions/greedysearch-dm/bin/mcp.mjs +52 -52
- package/extensions/greedysearch-dm/bin/search.mjs +130 -131
- package/extensions/greedysearch-dm/extractors/consensus.mjs +42 -43
- package/extensions/greedysearch-dm/extractors/gemini.mjs +52 -53
- package/extensions/greedysearch-dm/extractors/logically.mjs +33 -34
- package/extensions/greedysearch-dm/src/search/browser-lifecycle.mjs +9 -10
- package/extensions/greedysearch-dm/src/search/chrome.mjs +21 -22
- package/extensions/greedysearch-dm/src/search/engines.mjs +7 -7
- package/extensions/greedysearch-dm/src/search/fetch-source.mjs +48 -48
- package/extensions/greedysearch-dm/src/search/paths.mjs +1 -1
- package/extensions/greedysearch-dm/src/search/research.mjs +119 -119
- package/extensions/greedysearch-dm/src/search/scale-aware.mjs +7 -7
- package/extensions/greedysearch-dm/src/search/simple-research.mjs +125 -125
- package/extensions/greedysearch-dm/src/search/synthesis-runner.mjs +11 -11
- package/package.json +1 -1
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import{createRequire as On}from"node:module";var
|
|
1
|
+
import{createRequire as On}from"node:module";var V=On(import.meta.url);import{existsSync as fe,mkdirSync as Nn,readFileSync as It,writeFileSync as Pn}from"node:fs";import{homedir as xn}from"node:os";import{join as jt}from"node:path";import{tmpdir as In}from"node:os";function jn(e,t=9222){let r=Number.parseInt(String(e??""),10);return Number.isInteger(r)&&r>1024&&r<65535?r:t}var Tn=In().replaceAll("\\","/"),Yl=jn(process.env.GREEDY_SEARCH_PORT),k=(process.env.GREEDY_SEARCH_PROFILE_DIR||process.env.CDP_PROFILE_DIR||`${Tn}/greedysearch-chrome-profile`).replaceAll("\\","/"),Wl=`${k}/DevToolsActivePort`,Bl=process.env.GREEDY_SEARCH_PID_FILE||`${k}/browser.pid`,Ql=process.env.CDP_PAGES_CACHE||`${k}/cdp-pages.json`,Kl=process.env.GREEDY_SEARCH_MODE_FILE||`${k}/browser-mode`,Vl=process.env.GREEDY_SEARCH_METADATA_FILE||`${k}/browser-metadata.json`,zl=process.env.GREEDY_SEARCH_LAUNCH_LOCK_FILE||`${k}/browser-launch.lock`,Zl=process.env.GREEDY_SEARCH_ACTIVITY_FILE||`${k}/browser-last-activity`,Xl=(process.env.CDP_SOCKET_DIR||`${k}/cdp-sockets`).replaceAll("\\","/"),ec=`${k}/visible-recovery.jsonl`,He=jt(xn(),".dm"),q=jt(He,"greedyconfig"),qe=["perplexity","google","chatgpt","gemini"],Je="gemini";function Ln(){try{if(fe(q)){let e=It(q,"utf8"),t=JSON.parse(e);if(Array.isArray(t.engines)&&t.engines.length>0&&t.engines.every((r)=>typeof r==="string")){let r=t.engines.filter((i)=>Fe[i]),n=t.engines.filter((i)=>!Fe[i]);if(n.length>0)process.stderr.write(`[greedysearch] Warning: ignoring unknown engine(s) in ${q}: ${n.join(", ")}
|
|
2
2
|
[greedysearch] Available engines: ${Object.keys(Fe).join(", ")}
|
|
3
3
|
`);if(r.length>0)return r;process.stderr.write(`[greedysearch] Warning: no valid engines in ${q}, falling back to defaults: ${qe.join(", ")}
|
|
4
|
-
`)}}}catch{}return qe}function Mn(){try{if(!fe(He))Nn(He,{recursive:!0});if(!fe(q))
|
|
5
|
-
`,"utf8")}catch{}}Mn();var ae=["gemini","chatgpt"];function Un(){try{if(fe(q)){let
|
|
4
|
+
`)}}}catch{}return qe}function Mn(){try{if(!fe(He))Nn(He,{recursive:!0});if(!fe(q))Pn(q,JSON.stringify({engines:qe,synthesizer:Je},null,2)+`
|
|
5
|
+
`,"utf8")}catch{}}Mn();var ae=["gemini","chatgpt"];function Un(){try{if(fe(q)){let e=It(q,"utf8"),t=JSON.parse(e);if(typeof t.synthesizer==="string"){let r=t.synthesizer.toLowerCase();if(ae.includes(r))return r;process.stderr.write(`[greedysearch] Warning: unknown synthesizer "${t.synthesizer}" in ${q}
|
|
6
6
|
[greedysearch] Available synthesizers: ${ae.join(", ")}
|
|
7
7
|
[greedysearch] Falling back to default: ${Je}
|
|
8
|
-
`)}}}catch{}return Je}var Fe={perplexity:"perplexity.mjs",p:"perplexity.mjs",bing:"bing-copilot.mjs",b:"bing-copilot.mjs",google:"google-ai.mjs",g:"google-ai.mjs",gemini:"gemini.mjs",gem:"gemini.mjs",chatgpt:"chatgpt.mjs",gpt:"chatgpt.mjs","semantic-scholar":"semantic-scholar.mjs",semanticscholar:"semantic-scholar.mjs",s2:"semantic-scholar.mjs",logically:"logically.mjs",log:"logically.mjs"},pe=Ln(),Tt=pe,zl=Un(),Zl=Math.max(1,Number.parseInt(process.env.GREEDY_FETCH_CONCURRENCY||"5",10)||5);process.env.CDP_PROFILE_DIR=k;var Gn=["fbclid","gclid","ref","ref_src","ref_url","source","utm_campaign","utm_content","utm_medium","utm_source","utm_term"],Fn=["dev.to","hashnode.com","medium.com","reddit.com","stackoverflow.com","stackexchange.com","substack.com"],Hn=["arstechnica.com","techcrunch.com","theverge.com","venturebeat.com","wired.com","zdnet.com"],Mt=["facebook.com","instagram.com","linkedin.com","pinterest.com","tiktok.com","twitter.com","x.com"];function te(t="",e=240){let r=String(t).replaceAll(/\s+/g," ").trim();if(r.length<=e)return r;let n=r.slice(0,e),i=n.lastIndexOf(" ");return i>0?`${n.slice(0,i)}...`:`${n}...`}function Ye(t=""){let e=te(t,180);if(!e)return"";if(/^https?:\/\//i.test(e))return"";let r=e.split(/\s+/).filter(Boolean).length,n=/[A-Z]/.test(e),i=/\d/.test(e);return e===e.toLowerCase()&&r<=4&&!n&&!i?"":e}function We(t="",e=""){let r=Ye(t),n=Ye(e);if(!n)return r;if(!r)return n;let i=/^https?:\/\//i.test(r),a=/^https?:\/\//i.test(n);if(i&&!a)return n;if(!i&&a)return r;return n.length>r.length?n:r}function he(t){if(!t)return null;try{let e=new URL(t);if(!["http:","https:"].includes(e.protocol))return null;if(e.hash="",e.hostname=e.hostname.toLowerCase(),e.protocol==="https:"&&e.port==="443"||e.protocol==="http:"&&e.port==="80")e.port="";for(let i of[...e.searchParams.keys()]){let a=i.toLowerCase();if(Gn.includes(a)||a.startsWith("utm_"))e.searchParams.delete(i)}e.searchParams.sort();let r=e.pathname.replace(/\/{1,10}$/,"")||"/";e.pathname=r;let n=e.toString();return r==="/"?n.replace(/\/$/,""):n}catch{return null}}function qn(t){try{return new URL(t).hostname.toLowerCase().replace(/^www\./,"")}catch{return""}}function oe(t,e){return e.some((r)=>t===r||t.endsWith(`.${r}`))}function Jn(t,e="",r=""){let n=e.toLowerCase(),i=r.toLowerCase();if(t==="github.com"||t==="gitlab.com")return"repo";if(t==="arxiv.org"||t==="doi.org"||t==="semanticscholar.org"||t.endsWith(".semanticscholar.org")||i.includes("/paper/")||i.includes("/pdf/"))return"academic";if(oe(t,Mt))return"social";if(oe(t,Fn))return"community";if(oe(t,Hn))return"news";if(t.startsWith("docs.")||t.startsWith("developer.")||t.startsWith("developers.")||t.startsWith("api.")||n.includes("documentation")||n.includes("docs")||n.includes("reference")||i.includes("/docs/")||i.includes("/reference/")||i.includes("/api/"))return"official-docs";if(t.startsWith("blog.")||i.includes("/blog/"))return"maintainer-blog";return"website"}function Yn(t){switch(t){case"official-docs":return 5;case"repo":return 4;case"academic":return 4;case"maintainer-blog":return 3;case"website":return 2;case"community":return 1;case"news":return 0;case"social":return-6;default:return 0}}function Wn(t){let e=Object.values(t.perEngine||{}).map((r)=>r?.rank||99);return e.length?Math.min(...e):99}var Bn=["reddit.com","news.ycombinator.com","lobste.rs"];function ge(t){return t.smartScore*3+t.engineCount*5+Yn(t.sourceType)*2+Math.max(0,7-Wn(t))}function Qn(t){let e=t.toLowerCase(),r=[];if(e.includes("openai")||e.includes("gpt")||e.includes("chatgpt"))r.push("openai.com","platform.openai.com","help.openai.com");if(e.includes("anthropic")||e.includes("claude"))r.push("anthropic.com","docs.anthropic.com");if(e.includes("bun"))r.push("bun.sh","bun.com");if(e.includes("next.js")||e.includes("nextjs"))r.push("nextjs.org","vercel.com");if(e.includes("playwright"))r.push("playwright.dev");if(e.includes("supabase"))r.push("supabase.com","supabase.io");if(e.includes("prisma"))r.push("prisma.io");if(e.includes("tailwind"))r.push("tailwindcss.com");if(e.includes("vite"))r.push("vitejs.dev","vite.dev");if(e.includes("astro"))r.push("astro.build");if(e.includes("svelte"))r.push("svelte.dev");if(e.includes("solid"))r.push("solidjs.com");if(e.includes("vue")||e.includes("nuxt"))r.push("vuejs.org","nuxt.com");if(e.includes("react")||e.includes("react native"))r.push("react.dev","reactnative.dev");if(e.includes("angular"))r.push("angular.io","angular.dev");if(e.includes("node.js")||e.includes("nodejs"))r.push("nodejs.org","nodejs.dev","npmjs.com");if(/\bgo\b/.test(e)||e.includes("golang"))r.push("go.dev","golang.org","pkg.go.dev");if(e.includes("deno"))r.push("deno.land","deno.com");if(e.includes("fresh"))r.push("fresh.deno.dev");if(e.includes("typescript")||e.includes("ts"))r.push("typescriptlang.org");if(e.includes("python"))r.push("python.org","docs.python.org");if(e.includes("rust"))r.push("rust-lang.org","docs.rs","crates.io");if(e.includes("zig"))r.push("ziglang.org");if(e.includes("docker"))r.push("docker.com","docs.docker.com","hub.docker.com");if(e.includes("kubernetes")||e.includes("k8s"))r.push("kubernetes.io","k8s.io");if(e.includes("postgres")||e.includes("postgresql"))r.push("postgresql.org","neon.tech","supabase.com");if(e.includes("redis"))r.push("redis.io");if(e.includes("sqlite"))r.push("sqlite.org");if(e.includes("cloudflare"))r.push("developers.cloudflare.com","cloudflare.com");if(e.includes("vercel"))r.push("vercel.com","nextjs.org");if(e.includes("netlify"))r.push("netlify.com","docs.netlify.com");if(e.includes("stripe"))r.push("stripe.com","docs.stripe.com");if(e.includes("github"))r.push("github.com","docs.github.com");if(e.includes("gitlab"))r.push("gitlab.com","docs.gitlab.com");if(e.includes("aws"))r.push("aws.amazon.com","docs.aws.amazon.com");if(e.includes("azure"))r.push("azure.microsoft.com","learn.microsoft.com");if(e.includes("gcp")||e.includes("google cloud"))r.push("cloud.google.com","developers.google.com");if(e.includes("gemini")||e.includes("google ai"))r.push("ai.google.dev","developers.google.com");for(let n of Mt){let i=n.replace(/\.com$/,"");if(e.includes(i))r.push(n)}return[...new Set(r)]}function Lt(t,e){return t===e||t.endsWith(`.${e}`)}function Be(t,e=""){let r=new Map,n=Object.keys(t||{}).filter((c)=>!c.startsWith("_")),i=Qn(e);for(let c of n){let u=t[c];if(!u?.sources)continue;for(let d=0;d<u.sources.length;d++){let p=u.sources[d],f=he(p.url);if(!f||f.length<10)continue;let m=Ye(p.title||""),g=qn(f),h=Jn(g,m,f),y=0;if(i.some((E)=>Lt(g,E)))y+=10;if(h==="official-docs")y+=3;let v=f.toLowerCase();if(/\/docs\/|\/documentation\/|\.dev\/|\/api\/|\/reference\//.test(v))y+=2;let b=i.some((E)=>Lt(g,E));if(h==="social"&&!b)y-=20;if(i.length>0){if(oe(g,Bn))y-=3;else if(h==="community"&&!oe(g,["stackoverflow.com","stackexchange.com"]))y-=1}let w=r.get(f)||{id:"",canonicalUrl:f,displayUrl:p.url||f,domain:g,title:"",engines:[],engineCount:0,perEngine:{},sourceType:h,isOfficial:h==="official-docs",smartScore:0};if(w.title=We(w.title,m),w.displayUrl=w.displayUrl||p.url||f,w.sourceType=w.sourceType||h,w.isOfficial=w.isOfficial||h==="official-docs",w.smartScore=Math.max(w.smartScore,y),!w.engines.includes(c))w.engines.push(c);w.perEngine[c]={rank:d+1,title:We(w.perEngine[c]?.title||"",m)},r.set(f,w)}}let a=Array.from(r.values()).map((c)=>({...c,engineCount:c.engines.length})),o=a.filter((c)=>c.sourceType!=="social"),s=a.filter((c)=>c.sourceType==="social");return o.sort((c,u)=>{let d=ge(u)-ge(c);if(d!==0)return d;return c.domain.localeCompare(u.domain)}),s.sort((c,u)=>{let d=ge(u)-ge(c);if(d!==0)return d;return c.domain.localeCompare(u.domain)}),[...o,...s].slice(0,12).map((c,u)=>({...c,id:`S${u+1}`,title:c.title||c.domain||c.canonicalUrl}))}function Ut(t,e){let r=new Map(e.map((n)=>[n.id,n]));return t.map((n)=>{let i=r.get(n.id);if(!i)return n;let a=We(n.title,i.title||"");return{...n,title:a||n.title,fetch:{attempted:!0,ok:!i.error&&i.contentChars>100,status:i.status||null,finalUrl:i.finalUrl||i.url||n.canonicalUrl,contentType:i.contentType||"",lastModified:i.lastModified||"",publishedTime:i.publishedTime||"",byline:i.byline||"",siteName:i.siteName||"",lang:i.lang||"",title:i.title||"",snippet:i.snippet||"",contentChars:i.contentChars||0,source:i.source||"unknown",duration:i.duration||0,error:i.error||""}}})}import{createRequire as Vn}from"node:module";import{mkdirSync as Xn,writeFileSync as ei}from"node:fs";import{join as nr}from"node:path";import{createRequire as ni}from"node:module";import{createRequire as ii}from"node:module";import{spawn as ai}from"node:child_process";import{basename as oi}from"node:path";import{dirname as si,join as li}from"node:path";import{fileURLToPath as ci}from"node:url";import{existsSync as $e,mkdirSync as ui,readFileSync as or,writeFileSync as di}from"node:fs";import{homedir as fi}from"node:os";import{join as Gt}from"node:path";import{tmpdir as pi}from"node:os";import{existsSync as _e,mkdirSync as gi,readFileSync as sr,writeFileSync as hi}from"node:fs";import{homedir as mi}from"node:os";import{join as Ft}from"node:path";import{tmpdir as yi}from"node:os";import{existsSync as Ee,mkdirSync as wi,readFileSync as lr,writeFileSync as bi}from"node:fs";import{homedir as vi}from"node:os";import{join as Ht}from"node:path";import{tmpdir as $i}from"node:os";import{basename as eo}from"node:path";import{mkdirSync as Qe,writeFileSync as C}from"node:fs";import{join as _}from"node:path";import{fileURLToPath as ro}from"node:url";import{existsSync as Oe,mkdirSync as io,readFileSync as Or,writeFileSync as ao}from"node:fs";import{homedir as oo}from"node:os";import{join as Nr}from"node:path";import{tmpdir as so}from"node:os";import{spawn as mo}from"node:child_process";import{existsSync as Pr}from"node:fs";import{dirname as yo,join as ye}from"node:path";import{fileURLToPath as wo}from"node:url";import{join as Ao}from"node:path";import{fileURLToPath as Co}from"node:url";var Kn=Object.defineProperty,zn=(t)=>t;function Zn(t,e){this[t]=zn.bind(null,e)}var ut=(t,e)=>{for(var r in e)Kn(t,r,{get:e[r],enumerable:!0,configurable:!0,set:Zn.bind(e,r)})},dt=(t,e)=>()=>(t&&(e=t(t=0)),e),rc=Vn(import.meta.url),rr={};ut(rr,{writeSourcesToFiles:()=>ti});function ti(t,e=ir){return Xn(e,{recursive:!0}),t.map((r)=>{if(!r.content||r.content.length<10)return r;let n=String(r.id||"unknown").replace(/[^a-zA-Z0-9_-]/g,""),i=(r.canonicalUrl||r.url||"").replace(/^https?:\/\//,"").replace(/[^a-zA-Z0-9]/g,"-").slice(0,40),a=`${n}-${i}.md`,o=nr(e,a),s=`---
|
|
8
|
+
`)}}}catch{}return Je}var Fe={perplexity:"perplexity.mjs",p:"perplexity.mjs",bing:"bing-copilot.mjs",b:"bing-copilot.mjs",google:"google-ai.mjs",g:"google-ai.mjs",gemini:"gemini.mjs",gem:"gemini.mjs",chatgpt:"chatgpt.mjs",gpt:"chatgpt.mjs","semantic-scholar":"semantic-scholar.mjs",semanticscholar:"semantic-scholar.mjs",s2:"semantic-scholar.mjs",logically:"logically.mjs",log:"logically.mjs"},pe=Ln(),Tt=pe,tc=Un(),rc=Math.max(1,Number.parseInt(process.env.GREEDY_FETCH_CONCURRENCY||"5",10)||5);process.env.CDP_PROFILE_DIR=k;var Gn=["fbclid","gclid","ref","ref_src","ref_url","source","utm_campaign","utm_content","utm_medium","utm_source","utm_term"],Fn=["dev.to","hashnode.com","medium.com","reddit.com","stackoverflow.com","stackexchange.com","substack.com"],Hn=["arstechnica.com","techcrunch.com","theverge.com","venturebeat.com","wired.com","zdnet.com"],Mt=["facebook.com","instagram.com","linkedin.com","pinterest.com","tiktok.com","twitter.com","x.com"];function te(e="",t=240){let r=String(e).replaceAll(/\s+/g," ").trim();if(r.length<=t)return r;let n=r.slice(0,t),i=n.lastIndexOf(" ");return i>0?`${n.slice(0,i)}...`:`${n}...`}function Ye(e=""){let t=te(e,180);if(!t)return"";if(/^https?:\/\//i.test(t))return"";let r=t.split(/\s+/).filter(Boolean).length,n=/[A-Z]/.test(t),i=/\d/.test(t);return t===t.toLowerCase()&&r<=4&&!n&&!i?"":t}function We(e="",t=""){let r=Ye(e),n=Ye(t);if(!n)return r;if(!r)return n;let i=/^https?:\/\//i.test(r),a=/^https?:\/\//i.test(n);if(i&&!a)return n;if(!i&&a)return r;return n.length>r.length?n:r}function ge(e){if(!e)return null;try{let t=new URL(e);if(!["http:","https:"].includes(t.protocol))return null;if(t.hash="",t.hostname=t.hostname.toLowerCase(),t.protocol==="https:"&&t.port==="443"||t.protocol==="http:"&&t.port==="80")t.port="";for(let i of[...t.searchParams.keys()]){let a=i.toLowerCase();if(Gn.includes(a)||a.startsWith("utm_"))t.searchParams.delete(i)}t.searchParams.sort();let r=t.pathname.replace(/\/{1,10}$/,"")||"/";t.pathname=r;let n=t.toString();return r==="/"?n.replace(/\/$/,""):n}catch{return null}}function qn(e){try{return new URL(e).hostname.toLowerCase().replace(/^www\./,"")}catch{return""}}function oe(e,t){return t.some((r)=>e===r||e.endsWith(`.${r}`))}function Jn(e,t="",r=""){let n=t.toLowerCase(),i=r.toLowerCase();if(e==="github.com"||e==="gitlab.com")return"repo";if(e==="arxiv.org"||e==="doi.org"||e==="semanticscholar.org"||e.endsWith(".semanticscholar.org")||i.includes("/paper/")||i.includes("/pdf/"))return"academic";if(oe(e,Mt))return"social";if(oe(e,Fn))return"community";if(oe(e,Hn))return"news";if(e.startsWith("docs.")||e.startsWith("developer.")||e.startsWith("developers.")||e.startsWith("api.")||n.includes("documentation")||n.includes("docs")||n.includes("reference")||i.includes("/docs/")||i.includes("/reference/")||i.includes("/api/"))return"official-docs";if(e.startsWith("blog.")||i.includes("/blog/"))return"maintainer-blog";return"website"}function Yn(e){switch(e){case"official-docs":return 5;case"repo":return 4;case"academic":return 4;case"maintainer-blog":return 3;case"website":return 2;case"community":return 1;case"news":return 0;case"social":return-6;default:return 0}}function Wn(e){let t=Object.values(e.perEngine||{}).map((r)=>r?.rank||99);return t.length?Math.min(...t):99}var Bn=["reddit.com","news.ycombinator.com","lobste.rs"];function he(e){return e.smartScore*3+e.engineCount*5+Yn(e.sourceType)*2+Math.max(0,7-Wn(e))}function Qn(e){let t=e.toLowerCase(),r=[];if(t.includes("openai")||t.includes("gpt")||t.includes("chatgpt"))r.push("openai.com","platform.openai.com","help.openai.com");if(t.includes("anthropic")||t.includes("claude"))r.push("anthropic.com","docs.anthropic.com");if(t.includes("bun"))r.push("bun.sh","bun.com");if(t.includes("next.js")||t.includes("nextjs"))r.push("nextjs.org","vercel.com");if(t.includes("playwright"))r.push("playwright.dev");if(t.includes("supabase"))r.push("supabase.com","supabase.io");if(t.includes("prisma"))r.push("prisma.io");if(t.includes("tailwind"))r.push("tailwindcss.com");if(t.includes("vite"))r.push("vitejs.dev","vite.dev");if(t.includes("astro"))r.push("astro.build");if(t.includes("svelte"))r.push("svelte.dev");if(t.includes("solid"))r.push("solidjs.com");if(t.includes("vue")||t.includes("nuxt"))r.push("vuejs.org","nuxt.com");if(t.includes("react")||t.includes("react native"))r.push("react.dev","reactnative.dev");if(t.includes("angular"))r.push("angular.io","angular.dev");if(t.includes("node.js")||t.includes("nodejs"))r.push("nodejs.org","nodejs.dev","npmjs.com");if(/\bgo\b/.test(t)||t.includes("golang"))r.push("go.dev","golang.org","pkg.go.dev");if(t.includes("deno"))r.push("deno.land","deno.com");if(t.includes("fresh"))r.push("fresh.deno.dev");if(t.includes("typescript")||t.includes("ts"))r.push("typescriptlang.org");if(t.includes("python"))r.push("python.org","docs.python.org");if(t.includes("rust"))r.push("rust-lang.org","docs.rs","crates.io");if(t.includes("zig"))r.push("ziglang.org");if(t.includes("docker"))r.push("docker.com","docs.docker.com","hub.docker.com");if(t.includes("kubernetes")||t.includes("k8s"))r.push("kubernetes.io","k8s.io");if(t.includes("postgres")||t.includes("postgresql"))r.push("postgresql.org","neon.tech","supabase.com");if(t.includes("redis"))r.push("redis.io");if(t.includes("sqlite"))r.push("sqlite.org");if(t.includes("cloudflare"))r.push("developers.cloudflare.com","cloudflare.com");if(t.includes("vercel"))r.push("vercel.com","nextjs.org");if(t.includes("netlify"))r.push("netlify.com","docs.netlify.com");if(t.includes("stripe"))r.push("stripe.com","docs.stripe.com");if(t.includes("github"))r.push("github.com","docs.github.com");if(t.includes("gitlab"))r.push("gitlab.com","docs.gitlab.com");if(t.includes("aws"))r.push("aws.amazon.com","docs.aws.amazon.com");if(t.includes("azure"))r.push("azure.microsoft.com","learn.microsoft.com");if(t.includes("gcp")||t.includes("google cloud"))r.push("cloud.google.com","developers.google.com");if(t.includes("gemini")||t.includes("google ai"))r.push("ai.google.dev","developers.google.com");for(let n of Mt){let i=n.replace(/\.com$/,"");if(t.includes(i))r.push(n)}return[...new Set(r)]}function Lt(e,t){return e===t||e.endsWith(`.${t}`)}function Be(e,t=""){let r=new Map,n=Object.keys(e||{}).filter((c)=>!c.startsWith("_")),i=Qn(t);for(let c of n){let u=e[c];if(!u?.sources)continue;for(let d=0;d<u.sources.length;d++){let p=u.sources[d],f=ge(p.url);if(!f||f.length<10)continue;let m=Ye(p.title||""),h=qn(f),g=Jn(h,m,f),y=0;if(i.some((E)=>Lt(h,E)))y+=10;if(g==="official-docs")y+=3;let v=f.toLowerCase();if(/\/docs\/|\/documentation\/|\.dev\/|\/api\/|\/reference\//.test(v))y+=2;let b=i.some((E)=>Lt(h,E));if(g==="social"&&!b)y-=20;if(i.length>0){if(oe(h,Bn))y-=3;else if(g==="community"&&!oe(h,["stackoverflow.com","stackexchange.com"]))y-=1}let w=r.get(f)||{id:"",canonicalUrl:f,displayUrl:p.url||f,domain:h,title:"",engines:[],engineCount:0,perEngine:{},sourceType:g,isOfficial:g==="official-docs",smartScore:0};if(w.title=We(w.title,m),w.displayUrl=w.displayUrl||p.url||f,w.sourceType=w.sourceType||g,w.isOfficial=w.isOfficial||g==="official-docs",w.smartScore=Math.max(w.smartScore,y),!w.engines.includes(c))w.engines.push(c);w.perEngine[c]={rank:d+1,title:We(w.perEngine[c]?.title||"",m)},r.set(f,w)}}let a=Array.from(r.values()).map((c)=>({...c,engineCount:c.engines.length})),o=a.filter((c)=>c.sourceType!=="social"),s=a.filter((c)=>c.sourceType==="social");return o.sort((c,u)=>{let d=he(u)-he(c);if(d!==0)return d;return c.domain.localeCompare(u.domain)}),s.sort((c,u)=>{let d=he(u)-he(c);if(d!==0)return d;return c.domain.localeCompare(u.domain)}),[...o,...s].slice(0,12).map((c,u)=>({...c,id:`S${u+1}`,title:c.title||c.domain||c.canonicalUrl}))}function Ut(e,t){let r=new Map(t.map((n)=>[n.id,n]));return e.map((n)=>{let i=r.get(n.id);if(!i)return n;let a=We(n.title,i.title||"");return{...n,title:a||n.title,fetch:{attempted:!0,ok:!i.error&&i.contentChars>100,status:i.status||null,finalUrl:i.finalUrl||i.url||n.canonicalUrl,contentType:i.contentType||"",lastModified:i.lastModified||"",publishedTime:i.publishedTime||"",byline:i.byline||"",siteName:i.siteName||"",lang:i.lang||"",title:i.title||"",snippet:i.snippet||"",contentChars:i.contentChars||0,source:i.source||"unknown",duration:i.duration||0,error:i.error||""}}})}import{createRequire as Kn}from"node:module";import{mkdirSync as Xn,writeFileSync as ei}from"node:fs";import{join as nr}from"node:path";import{createRequire as ni}from"node:module";import{createRequire as ii}from"node:module";import{spawn as ai}from"node:child_process";import{basename as oi}from"node:path";import{dirname as si,join as li}from"node:path";import{fileURLToPath as ci}from"node:url";import{fileURLToPath as ui}from"node:url";import{existsSync as $e,mkdirSync as di,readFileSync as or,writeFileSync as fi}from"node:fs";import{homedir as pi}from"node:os";import{join as Gt}from"node:path";import{tmpdir as hi}from"node:os";import{existsSync as _e,mkdirSync as gi,readFileSync as sr,writeFileSync as mi}from"node:fs";import{homedir as yi}from"node:os";import{join as Ft}from"node:path";import{tmpdir as wi}from"node:os";import{existsSync as Ee,mkdirSync as bi,readFileSync as lr,writeFileSync as vi}from"node:fs";import{homedir as $i}from"node:os";import{join as Ht}from"node:path";import{tmpdir as _i}from"node:os";import{basename as ro}from"node:path";import{mkdirSync as Qe,writeFileSync as C}from"node:fs";import{join as _}from"node:path";import{fileURLToPath as io}from"node:url";import{existsSync as Oe,mkdirSync as oo,readFileSync as Or,writeFileSync as so}from"node:fs";import{homedir as lo}from"node:os";import{join as Nr}from"node:path";import{tmpdir as co}from"node:os";import{spawn as wo}from"node:child_process";import{existsSync as xr}from"node:fs";import{dirname as bo,join as ye}from"node:path";import{fileURLToPath as vo}from"node:url";import{join as ko}from"node:path";import{fileURLToPath as Oo}from"node:url";var Vn=Object.defineProperty,zn=(e)=>e;function Zn(e,t){this[e]=zn.bind(null,t)}var ut=(e,t)=>{for(var r in t)Vn(e,r,{get:t[r],enumerable:!0,configurable:!0,set:Zn.bind(t,r)})},dt=(e,t)=>()=>(e&&(t=e(e=0)),t),oc=Kn(import.meta.url),rr={};ut(rr,{writeSourcesToFiles:()=>ti});function ti(e,t=ir){return Xn(t,{recursive:!0}),e.map((r)=>{if(!r.content||r.content.length<10)return r;let n=String(r.id||"unknown").replace(/[^a-zA-Z0-9_-]/g,""),i=(r.canonicalUrl||r.url||"").replace(/^https?:\/\//,"").replace(/[^a-zA-Z0-9]/g,"-").slice(0,40),a=`${n}-${i}.md`,o=nr(t,a),s=`---
|
|
9
9
|
url: ${r.finalUrl||r.url}
|
|
10
10
|
title: ${r.title||""}
|
|
11
11
|
source: ${r.source||"unknown"}
|
|
@@ -13,49 +13,49 @@ status: ${r.status||""}
|
|
|
13
13
|
chars: ${r.contentChars||r.content.length}
|
|
14
14
|
---
|
|
15
15
|
|
|
16
|
-
`;ei(o,s+r.content,"utf8");let{content:l,...c}=r;return{...c,contentPath:o,contentChars:r.contentChars||l.length}})}var ir,ar=dt(()=>{ir=nr(process.cwd(),".dm","greedysearch-sources")}),ri={};ut(ri,{fetchTopSource:()=>
|
|
16
|
+
`;ei(o,s+r.content,"utf8");let{content:l,...c}=r;return{...c,contentPath:o,contentChars:r.contentChars||l.length}})}var ir,ar=dt(()=>{ir=nr(process.cwd(),".dm","greedysearch-sources")}),ri={};ut(ri,{fetchTopSource:()=>oa,fetchSourceContent:()=>yr,fetchMultipleSources:()=>aa});async function Ei(){if(me)return me;let[{Readability:e},{JSDOM:t},{default:r}]=await Promise.all([import("@mozilla/readability"),import("jsdom"),import("turndown")]),n=new r({headingStyle:"atx",bulletListMarker:"-",codeBlockStyle:"fenced"});return n.addRule("removeDataUrls",{filter:(i)=>i.tagName==="IMG"&&i.getAttribute("src")?.startsWith("data:"),replacement:()=>""}),me={Readability:e,JSDOM:t,turndown:n},me}function Si(e){if(!e||e.includes(":"))return null;let t=e.split(".");if(t.length<1||t.length>4)return null;if(!t.every((i)=>_r.test(i)))return null;let r=t.map((i)=>{if(/^0x/i.test(i))return parseInt(i,16);if(/^0[0-7]+$/.test(i))return parseInt(i,8);return parseInt(i,10)});if(r.some((i)=>!Number.isFinite(i)||i<0))return null;let n;if(r.length===1){if(r[0]>4294967295)return null;n=[r[0]>>>24&255,r[0]>>>16&255,r[0]>>>8&255,r[0]&255]}else if(r.length===2){if(r[0]>255||r[1]>16777215)return null;n=[r[0],r[1]>>>16&255,r[1]>>>8&255,r[1]&255]}else if(r.length===3){if(r[0]>255||r[1]>255||r[2]>65535)return null;n=[r[0],r[1],r[2]>>>8&255,r[2]&255]}else{if(r.some((i)=>i>255))return null;n=r}return n.join(".")}function Ri(e){let t=e.match(vr);if(t)return`${t[1]}.${t[2]}.${t[3]}.${t[4]}`;let r=e.match($r);if(r){let n=parseInt(r[1],16),i=parseInt(r[2],16);if(n>65535||i>65535)return null;return[n>>>8&255,n&255,i>>>8&255,i&255].join(".")}return null}function qt(e){return gt.some((t)=>t.test(e))}function Di(e={}){return{...ht,...e}}function ze(e){try{if(typeof e!=="string"||!e.trim())return{blocked:!0,reason:"URL must be a non-empty string"};let t=new URL(e);if(t.protocol!=="http:"&&t.protocol!=="https:")return{blocked:!0,reason:`Protocol not allowed: ${t.protocol}`};let r=t.hostname.toLowerCase();for(let i of gt)if(i.test(r))return{blocked:!0,reason:`Private/internal address: ${r}`};if(r.startsWith("[")&&r.endsWith("]")){let i=r.slice(1,-1),a=Ri(i);if(a&&qt(a))return{blocked:!0,reason:`Private/internal address: ${r} (maps to ${a})`}}let n=Si(r);if(n&&qt(n))return{blocked:!0,reason:`Private/internal address: ${r} (normalizes to ${n})`};return{blocked:!1}}catch(t){return{blocked:!0,reason:`Invalid URL: ${t.message}`}}}function Ai(e){try{let t=new URL(e);if(!(t.hostname==="github.com"||t.hostname.endsWith(".github.com")))return e;let r=t.pathname.split("/").filter(Boolean);if(r.length<5)return e;let[n,i,a,o,...s]=r;if(a!=="blob")return e;let l=s.join("/");return`https://raw.githubusercontent.com/${n}/${i}/${o}/${l}`}catch{return e}}async function Ci(e,t={}){let r=ze(e);if(r.blocked)return{ok:!1,url:e,finalUrl:e,status:403,error:`Blocked: ${r.reason}`,needsBrowser:!1};let n=e;if(e=Ai(e),e!==n)console.error(`[fetcher] Rewrote GitHub URL: ${n.slice(0,60)}... → raw.githubusercontent.com`);let{timeoutMs:i=15000,userAgent:a,signal:o}=t,s=new AbortController,l=setTimeout(()=>s.abort(),i);if(o)o.addEventListener("abort",()=>s.abort(),{once:!0});try{let c=await fetch(e,{method:"GET",headers:{...ht,"user-agent":a||br},redirect:"follow",signal:s.signal});clearTimeout(l);let u=c.headers.get("content-type")||"",d=c.url,p=c.headers.get("last-modified")||"",f=ze(d);if(f.blocked)return{ok:!1,url:e,finalUrl:d,status:c.status,error:`Blocked: ${f.reason}`,needsBrowser:!1};let m=!1;try{m=new URL(d).hostname.toLowerCase()==="raw.githubusercontent.com"}catch{}if(u.includes("text/plain")&&m){let b=await c.text();return{ok:!0,url:n,finalUrl:d,status:c.status,title:d.split("/").pop()||"GitHub File",byline:"",siteName:"GitHub",lang:"",publishedTime:p,lastModified:p,markdown:b,contentLength:b.length,excerpt:b.slice(0,300).replaceAll(/\n/g," "),needsBrowser:!1}}if(!u.includes("text/html")&&!u.includes("application/xhtml"))return{ok:!1,url:e,finalUrl:d,status:c.status,error:`Unsupported content type: ${u}`,needsBrowser:!1};let h=await c.text(),g=cr(c.status,h,d,e);if(g.blocked)return{ok:!1,url:e,finalUrl:d,status:c.status,error:`Blocked: ${g.reason}`,needsBrowser:!0};let y=await ur(h,d),v=dr(y);if(!v.ok)return{ok:!1,url:e,finalUrl:d,status:c.status,error:`Low quality content: ${v.reason}`,needsBrowser:!0};return{ok:!0,url:e,finalUrl:d,status:c.status,title:y.title,byline:y.byline,siteName:y.siteName,lang:y.lang,publishedTime:y.publishedTime||p,lastModified:p,markdown:y.markdown,excerpt:y.excerpt,contentLength:y.markdown.length,needsBrowser:!1}}catch(c){clearTimeout(l);let u=Oi(c);return{ok:!1,url:e,finalUrl:e,status:0,error:c.message,needsBrowser:u}}}function cr(e,t,r,n){let i=t.match(/<title[^>]*>([^<]*)<\/title>/i)?.[1]?.toLowerCase()||"",a=t.slice(0,30000).toLowerCase(),o=`${i} ${a}`;if(e===403||e===429||e===503)return{blocked:!0,reason:`HTTP ${e}`};let s=[{pattern:/class=["'][^"']*captcha["']|<div[^>]*id=["']captcha/i,reason:"captcha"},{pattern:/g-recaptcha|data-sitekey|i['"]m not a robot/i,reason:"captcha"},{pattern:/checking your browser.{0,100}please wait|cf-browser-verification/i,reason:"cloudflare challenge"},{pattern:/just a moment.{0,50}security check|ddos protection by cloudflare/i,reason:"cloudflare challenge"},{pattern:/unusual traffic.{0,50}from your computer network/i,reason:"unusual traffic"},{pattern:/bot detected|automated.{0,20}request/i,reason:"bot detection"},{pattern:/enable\s+javascript\s+to\s+view|javascript\s+is\s+required.{0,50}enabled/i,reason:"requires javascript"},{pattern:/access denied|accessdenied/i,reason:"access denied"},{pattern:/protected by anubis|anubis uses a proof-of-work/i,reason:"anubis challenge"}];for(let c of s)if(c.pattern.test(o))return{blocked:!0,reason:c.reason};let l=ki(n,r,t);if(l)return{blocked:!0,reason:l};return{blocked:!1}}function ki(e,t,r){try{let n=new URL(e),i=new URL(t);if(n.hostname.toLowerCase()===i.hostname.toLowerCase())return;let a=i.hostname.toLowerCase();if(Er.some((s)=>a===s||a.endsWith(`.${s}`)))return`redirected to login (${i.hostname})`;if(Sr.some((s)=>a.startsWith(s)))return`redirected to login (${i.hostname})`;let o=r.slice(0,20000).toLowerCase();if(Rr.some((s)=>o.includes(s)))return`redirected to login page (${i.hostname})`}catch{}return}function Oi(e){let t=e.message.toLowerCase();return t.includes("fetch failed")||t.includes("unable to verify")||t.includes("certificate")||t.includes("timeout")}function Jt(e){let t=['meta[property="article:published_time"]','meta[name="article:published_time"]','meta[property="og:published_time"]','meta[name="publication_date"]','meta[name="date"]','meta[itemprop="datePublished"]','time[itemprop="datePublished"]','meta[name="DC.date"]'];for(let r of t){let n=e.querySelector(r),i=n?.getAttribute("content")||n?.getAttribute("datetime")||"";if(i)return i}return""}async function ur(e,t){let{Readability:r,JSDOM:n,turndown:i}=await Ei(),a=new n(e,{url:t});try{let o=a.window.document,s=new r(o).parse();if(s&&s.content){let c=i.turndown(s.content).replaceAll(/\n{3,}/g,`
|
|
17
17
|
|
|
18
|
-
`).trim(),u=s.publishedTime||Jt(o)||"";return{title:s.title||o.title||
|
|
18
|
+
`).trim(),u=s.publishedTime||Jt(o)||"";return{title:s.title||o.title||t,byline:s.byline||"",siteName:s.siteName||"",lang:s.lang||"",publishedTime:u,markdown:c,excerpt:c.slice(0,300).replaceAll(/\n/g," ")}}let l=o.body;if(l){let c=l.cloneNode(!0);c.querySelectorAll("script, style, nav, footer, header, aside").forEach((d)=>d.remove());let u=(c.textContent||"").replaceAll(/\s+/g," ").trim();return{title:o.title||t,byline:"",siteName:"",lang:"",publishedTime:Jt(o),markdown:u,excerpt:u.slice(0,300)}}return{title:t,byline:"",siteName:"",lang:"",publishedTime:"",markdown:"",excerpt:""}}finally{a.window.close()}}function dr(e){let t=e.markdown.trim().toLowerCase(),r=(e.title||"").toLowerCase();if(e.markdown.trim().length<100)return{ok:!1,reason:"content too short (< 100 chars)"};let n=t.toLowerCase(),i=[{check:()=>n.includes("loading")&&n.includes("please wait"),desc:"loading page"},{check:()=>n.includes("please ensure javascript is enabled"),desc:"requires javascript"},{check:()=>n.includes("enable javascript to view"),desc:"requires javascript"},{check:()=>n.includes("just a moment"),desc:"cloudflare challenge detected in content"},{check:()=>n.includes("verify you are human"),desc:"human verification"},{check:()=>n.includes("captcha required"),desc:"captcha in extracted content"},{check:()=>n.includes("access denied"),desc:"access denied in content"},{check:()=>/^\s{0,10}sign\s{1,5}in\s{0,10}$|^\s{0,10}log\s{1,5}in\s{0,10}$/im.test(t),desc:"login form only"}];for(let{check:a,desc:o}of i)if(a())return{ok:!1,reason:o};if(r.includes("just a moment")||r.includes("checking your browser"))return{ok:!1,reason:"cloudflare challenge page detected in title"};return{ok:!0}}function Ze(e){try{let t=new URL(e);if(!(t.hostname==="github.com"||t.hostname.endsWith(".github.com")))return null;let r=t.pathname.split("/").filter(Boolean);if(r.length<2)return null;let[n,i]=r;if(r.length===2)return{owner:n,repo:i,type:"root"};if(r.length>=4&&(r[2]==="blob"||r[2]==="tree")){let a=r[2],o=r[3],s=r.slice(4).join("/");return{owner:n,repo:i,type:a,ref:o,path:s}}return null}catch{return null}}async function x(e,t=1e4){let r=new AbortController,n=setTimeout(()=>r.abort(),t);try{let i=await fetch(`https://api.github.com${e}`,{headers:mt,signal:r.signal});if(clearTimeout(n),!i.ok)throw Error(`GitHub API ${i.status}: ${e}`);return await i.json()}catch(i){throw clearTimeout(n),i}}async function Ni(e,t){try{let r=await x(`/repos/${e}/${t}/readme`);if(r.content&&r.encoding==="base64")return Buffer.from(r.content,"base64").toString("utf8");return""}catch{return""}}async function Yt(e,t,r="HEAD",n="",i){try{let a;if(r==="HEAD")if(i)a=await x(`/repos/${e}/${t}/git/ref/heads/${i}`).catch(()=>null);else a=await Promise.any([x(`/repos/${e}/${t}/git/ref/heads/main`),x(`/repos/${e}/${t}/git/ref/heads/master`)]).catch(()=>null);else a=await x(`/repos/${e}/${t}/git/ref/heads/${r}`).catch(()=>x(`/repos/${e}/${t}/git/ref/heads/master`).catch(()=>null));if(!a?.object?.sha)return[];let o=(await x(`/repos/${e}/${t}/git/commits/${a.object.sha}`)).tree.sha,s=(await x(`/repos/${e}/${t}/git/trees/${o}`)).tree||[];if(n)s=s.filter((l)=>l.path.startsWith(n));return s.slice(0,50).map((l)=>({path:l.path,type:l.type==="tree"?"dir":"file",size:l.size}))}catch{return[]}}async function Pi(e,t,r,n,i=1e4,a){let o=async(l)=>{let c=new AbortController,u=setTimeout(()=>c.abort(),i);try{let d=await fetch(l,{headers:{"user-agent":mt["user-agent"]},signal:c.signal});if(clearTimeout(u),d.ok)return await d.text();throw Error("not ok")}catch{throw clearTimeout(u),Error("failed")}};if(!r||r==="HEAD"){if(a)try{return await o(`https://raw.githubusercontent.com/${e}/${t}/${a}/${n}`)}catch{return null}try{return await Promise.any([o(`https://raw.githubusercontent.com/${e}/${t}/main/${n}`),o(`https://raw.githubusercontent.com/${e}/${t}/master/${n}`)])}catch{return null}}let s=[`https://raw.githubusercontent.com/${e}/${t}/${r}/${n}`,`https://raw.githubusercontent.com/${e}/${t}/master/${n}`];for(let l of s)try{return await o(l)}catch{}return null}async function xi(e){let t=Ze(e);if(!t)return{ok:!1,error:"Not a valid GitHub URL"};let{owner:r,repo:n,type:i,ref:a,path:o}=t;try{if(i==="root"||i==="tree"&&!o){let s=await x(`/repos/${r}/${n}`),[l,c]=await Promise.allSettled([Ni(r,n),Yt(r,n,a||"HEAD","",s.default_branch)]),u=l.status==="fulfilled"?l.value:"",d=c.status==="fulfilled"?c.value:[],p=s?.description?`
|
|
19
19
|
|
|
20
|
-
> ${s.description}`:"",f=s?.stargazers_count==null?"":` ⭐ ${s.stargazers_count}`,m=s?.language?` · ${s.language}`:"",
|
|
20
|
+
> ${s.description}`:"",f=s?.stargazers_count==null?"":` ⭐ ${s.stargazers_count}`,m=s?.language?` · ${s.language}`:"",h=`# ${r}/${n}${f}${m}${p}
|
|
21
21
|
|
|
22
|
-
`;if(u)
|
|
22
|
+
`;if(u)h+=u.slice(0,6000);else h+=`[No README found]
|
|
23
23
|
|
|
24
24
|
Files:
|
|
25
|
-
${d.map((
|
|
26
|
-
`)}`;return{ok:!0,title:`${r}/${n}`,content:
|
|
25
|
+
${d.map((g)=>` ${g.type==="dir"?"\uD83D\uDCC1":"\uD83D\uDCC4"} ${g.path}`).join(`
|
|
26
|
+
`)}`;return{ok:!0,title:`${r}/${n}`,content:h,tree:d.slice(0,30)}}if(i==="blob"&&o){let s;if(!a||a==="HEAD")try{s=(await x(`/repos/${r}/${n}`)).default_branch}catch{s=void 0}let l=await Pi(r,n,a,o,1e4,s);if(l===null)return{ok:!1,error:`File not found: ${o}`};return{ok:!0,title:`${r}/${n}: ${o}`,content:l}}if(i==="tree"&&o){let s;if(!a||a==="HEAD")try{s=(await x(`/repos/${r}/${n}`)).default_branch}catch{s=void 0}let l=await Yt(r,n,a||"HEAD",o,s),c=l.map((u)=>` ${u.type==="dir"?"\uD83D\uDCC1":"\uD83D\uDCC4"} ${u.path}`).join(`
|
|
27
27
|
`);return{ok:!0,title:`${r}/${n}/${o}`,content:`[Directory: ${o}]
|
|
28
28
|
|
|
29
29
|
Files:
|
|
30
|
-
${c}`,tree:l}}return{ok:!1,error:"Unsupported GitHub URL type"}}catch(s){return{ok:!1,error:s.message}}}function
|
|
30
|
+
${c}`,tree:l}}return{ok:!1,error:"Unsupported GitHub URL type"}}catch(s){return{ok:!1,error:s.message}}}function Ii(e){try{let t=new URL(e),r=t.hostname.toLowerCase();if(!(r==="reddit.com"||r.endsWith(".reddit.com")))return null;let n=t.pathname;if(n.match(/^\/(u|user)\/[^/]+\/?$/i))return{type:"user",cleanUrl:Wt(e)};if(n.match(/^\/r\/[^/]+\/comments\/[^/]+/i))return{type:"post",cleanUrl:Wt(e)};return null}catch{return null}}function Wt(e){try{let t=new URL(e);return`${t.protocol}//${t.hostname}${t.pathname}`}catch{return e}}async function ji(e,t=8000){let r=Date.now();try{let n=e.replace(/\/+$/,"")+".json",i=new AbortController,a=setTimeout(()=>i.abort(),15000),o=await fetch(n,{headers:Dr,signal:i.signal});if(clearTimeout(a),!o.ok)throw Error(`Reddit API ${o.status}`);let s=await o.json();if(!Array.isArray(s)||s.length<1)throw Error("Invalid Reddit API response structure");let l=s[0],c=s[1],u=l?.data?.children?.[0]?.data;if(!u)throw Error("No post data in Reddit response");let d=Ti(u,c,t);return{ok:!0,url:e,finalUrl:e,status:200,contentType:"text/markdown",lastModified:"",title:u.title||"Reddit Post",byline:`u/${u.author}`,siteName:`r/${u.subreddit}`,lang:"en",publishedTime:new Date(u.created_utc*1000).toISOString(),excerpt:u.selftext?.slice(0,300).replace(/\n/g," ")||"",markdown:d,contentLength:d.length,needsBrowser:!1,duration:Date.now()-r}}catch(n){return{ok:!1,url:e,finalUrl:e,status:0,error:`Reddit fetch failed: ${n.message}`,needsBrowser:!1,duration:Date.now()-r}}}function Ti(e,t,r){let n="";if(n+=`# ${e.title}
|
|
31
31
|
|
|
32
|
-
`,n+=`**Subreddit:** r/${
|
|
32
|
+
`,n+=`**Subreddit:** r/${e.subreddit} | **Author:** u/${e.author} | **Score:** ${e.score}
|
|
33
33
|
|
|
34
|
-
`,
|
|
34
|
+
`,e.selftext)n+=e.selftext,n+=`
|
|
35
35
|
|
|
36
|
-
`;else if(
|
|
36
|
+
`;else if(e.url)try{let i=new URL(e.url).hostname.toLowerCase();if(i!=="reddit.com"&&!i.endsWith(".reddit.com"))n+=`**Link:** ${e.url}
|
|
37
37
|
|
|
38
|
-
`}catch{n+=`**Link:** ${
|
|
38
|
+
`}catch{n+=`**Link:** ${e.url}
|
|
39
39
|
|
|
40
|
-
`}if(
|
|
40
|
+
`}if(t?.data?.children?.length>0){n+=`---
|
|
41
41
|
|
|
42
42
|
## Comments
|
|
43
43
|
|
|
44
|
-
`;let i=
|
|
44
|
+
`;let i=t.data.children.filter((a)=>a.kind==="t1").slice(0,10);for(let a of i)n+=fr(a.data,0),n+=`
|
|
45
45
|
`}if(n.length>r)n=n.slice(0,r).trim()+`
|
|
46
46
|
|
|
47
|
-
... (truncated)`;return n}function fr(t
|
|
48
|
-
`,n+=`${r}${
|
|
47
|
+
... (truncated)`;return n}function fr(e,t){if(!e||e.body==="[deleted]"||e.body==="[removed]")return"";let r="> ".repeat(t),n="";if(n+=`${r}**u/${e.author}** (${e.score} pts)
|
|
48
|
+
`,n+=`${r}${e.body.replaceAll(`
|
|
49
49
|
`,`
|
|
50
50
|
`+r)}
|
|
51
|
-
`,
|
|
52
|
-
`+fr(a.data,
|
|
51
|
+
`,t<3&&e.replies?.data?.children){let i=e.replies.data.children.filter((a)=>a.kind==="t1");for(let a of i.slice(0,5))n+=`
|
|
52
|
+
`+fr(a.data,t+1)}return n}function re(e,t=8000){if(!e||e.length<=t)return e;let r=`
|
|
53
53
|
|
|
54
54
|
[...content trimmed...]
|
|
55
55
|
|
|
56
|
-
`,n=
|
|
57
|
-
`)o--;if(o<=i-100)o=i;let s=
|
|
58
|
-
`)s++;if(s>=
|
|
56
|
+
`,n=t-r.length,i=Math.floor(n*0.75),a=n-i,o=i;while(o>i-100&&e[o]!==`
|
|
57
|
+
`)o--;if(o<=i-100)o=i;let s=e.length-a;while(s<e.length-a+100&&e[s]!==`
|
|
58
|
+
`)s++;if(s>=e.length-a+100)s=e.length-a;let l=e.slice(0,o).trimEnd(),c=e.slice(s).trimStart();return`${l}${r}${c}`}function Li(e=process.env,t=process.execPath){let r=e.GREEDY_SEARCH_NODE||e.NODE_BINARY||e.NODE;if(r?.trim())return r.trim();let n=oi(t||"").toLowerCase();if(n==="node"||n==="node.exe")return t;return"node"}function Mi(e){if(!Array.isArray(e)||e.length===0)throw Error("cdp: args must be a non-empty array");if(e[0]==="test")return e.map((t,r)=>Bt(t,r));if(!Cr.has(e[0]))throw Error(`cdp: unknown subcommand '${e[0]}'`);return e.map((t,r)=>Bt(t,r))}function Bt(e,t){if(typeof e!=="string")throw Error(`cdp: argv[${t}] must be a string (got ${typeof e})`);if(e.includes("\x00"))throw Error(`cdp: argv[${t}] contains a null byte`);return e}function pr(e,t=30000){return Ui(e,null,t)}function Ui(e,t=null,r=30000){let n=Mi(e);return new Promise((i,a)=>{let o=ai(Li(),[Ar,...n],{stdio:[t==null?"ignore":"pipe","pipe","pipe"]});if(t!=null)o.stdin.write(t),o.stdin.end();let s="",l="";o.stdout.on("data",(u)=>s+=u),o.stderr.on("data",(u)=>l+=u);let c=setTimeout(()=>{o.kill(),a(Error(`cdp timeout: ${e[0]}`))},r);o.on("close",(u)=>{if(clearTimeout(c),u===0)i(s.trim());else a(Error(l.trim()||`cdp exit ${u}`))})})}async function Qt(e){await pr(["evalraw",e,"Page.addScriptToEvaluateOnNewDocument",JSON.stringify({source:`
|
|
59
59
|
(function() {
|
|
60
60
|
// ── Runtime.enable / CDP detection masking ──────────────
|
|
61
61
|
try { delete window.__REBROWSER_RUNTIME_ENABLE; } catch(_) {}
|
|
@@ -340,35 +340,35 @@ ${c}`,tree:l}}return{ok:!1,error:"Unsupported GitHub URL type"}}catch(s){return{
|
|
|
340
340
|
};
|
|
341
341
|
} catch(_) {}
|
|
342
342
|
})();
|
|
343
|
-
`})])}function
|
|
343
|
+
`})])}function Gi(e){return ui(new URL(".",e)).replace(/^\/([A-Z]:)/,"$1")}function Fi(e,t=9222){let r=Number.parseInt(String(e??""),10);return Number.isInteger(r)&&r>1024&&r<65535?r:t}function Hi(){try{if($e(L)){let e=or(L,"utf8"),t=JSON.parse(e);if(Array.isArray(t.engines)&&t.engines.length>0&&t.engines.every((r)=>typeof r==="string")){let r=t.engines.filter((i)=>we[i]),n=t.engines.filter((i)=>!we[i]);if(n.length>0)process.stderr.write(`[greedysearch] Warning: ignoring unknown engine(s) in ${L}: ${n.join(", ")}
|
|
344
344
|
[greedysearch] Available engines: ${Object.keys(we).join(", ")}
|
|
345
345
|
`);if(r.length>0)return r;process.stderr.write(`[greedysearch] Warning: no valid engines in ${L}, falling back to defaults: ${Re.join(", ")}
|
|
346
|
-
`)}}}catch{}return Re}function
|
|
347
|
-
`,"utf8")}catch{}}function
|
|
346
|
+
`)}}}catch{}return Re}function qi(){try{if(!$e(Se))di(Se,{recursive:!0});if(!$e(L))fi(L,JSON.stringify({engines:Re,synthesizer:Xe},null,2)+`
|
|
347
|
+
`,"utf8")}catch{}}function Ji(){try{if($e(L)){let e=or(L,"utf8"),t=JSON.parse(e);if(typeof t.synthesizer==="string"){let r=t.synthesizer.toLowerCase();if(et.includes(r))return r;process.stderr.write(`[greedysearch] Warning: unknown synthesizer "${t.synthesizer}" in ${L}
|
|
348
348
|
[greedysearch] Available synthesizers: ${et.join(", ")}
|
|
349
349
|
[greedysearch] Falling back to default: ${Xe}
|
|
350
|
-
`)}}}catch{}return Xe}function
|
|
350
|
+
`)}}}catch{}return Xe}function Yi(e,t=9222){let r=Number.parseInt(String(e??""),10);return Number.isInteger(r)&&r>1024&&r<65535?r:t}function Wi(){try{if(_e(M)){let e=sr(M,"utf8"),t=JSON.parse(e);if(Array.isArray(t.engines)&&t.engines.length>0&&t.engines.every((r)=>typeof r==="string")){let r=t.engines.filter((i)=>be[i]),n=t.engines.filter((i)=>!be[i]);if(n.length>0)process.stderr.write(`[greedysearch] Warning: ignoring unknown engine(s) in ${M}: ${n.join(", ")}
|
|
351
351
|
[greedysearch] Available engines: ${Object.keys(be).join(", ")}
|
|
352
352
|
`);if(r.length>0)return r;process.stderr.write(`[greedysearch] Warning: no valid engines in ${M}, falling back to defaults: ${Ae.join(", ")}
|
|
353
|
-
`)}}}catch{}return Ae}function
|
|
354
|
-
`,"utf8")}catch{}}function
|
|
353
|
+
`)}}}catch{}return Ae}function Bi(){try{if(!_e(De))gi(De,{recursive:!0});if(!_e(M))mi(M,JSON.stringify({engines:Ae,synthesizer:tt},null,2)+`
|
|
354
|
+
`,"utf8")}catch{}}function Qi(){try{if(_e(M)){let e=sr(M,"utf8"),t=JSON.parse(e);if(typeof t.synthesizer==="string"){let r=t.synthesizer.toLowerCase();if(rt.includes(r))return r;process.stderr.write(`[greedysearch] Warning: unknown synthesizer "${t.synthesizer}" in ${M}
|
|
355
355
|
[greedysearch] Available synthesizers: ${rt.join(", ")}
|
|
356
356
|
[greedysearch] Falling back to default: ${tt}
|
|
357
|
-
`)}}}catch{}return tt}async function
|
|
358
|
-
`)[0];if(!
|
|
357
|
+
`)}}}catch{}return tt}async function hr(){let e=(await R(["list"])).split(`
|
|
358
|
+
`)[0];if(!e)throw Error("No Chrome tabs found");return e.slice(0,8)}async function ft(e="about:blank"){let t=await hr(),r=new URL(e).hostname;if(r==="copilot.microsoft.com"||r==="www.perplexity.ai"||r==="perplexity.ai"||r.endsWith(".perplexity.ai")){let a=await R(["evalraw",t,"Target.createTarget",JSON.stringify({url:"about:blank"})]),{targetId:o}=JSON.parse(a),s=o.slice(0,8);if(await R(["list"]).catch(()=>null),r==="copilot.microsoft.com")await Qt(s);else Qt(s).catch(()=>{});return await R(["list"]).catch(()=>null),o}let n=await R(["evalraw",t,"Target.createTarget",JSON.stringify({url:e})]),{targetId:i}=JSON.parse(n);return await R(["list"]).catch(()=>null),i}async function pt(e){try{let t=await hr();await R(["evalraw",t,"Target.closeTarget",JSON.stringify({targetId:e})])}catch{}}function Ki(e,t=9222){let r=Number.parseInt(String(e??""),10);return Number.isInteger(r)&&r>1024&&r<65535?r:t}function Vi(){try{if(Ee(U)){let e=lr(U,"utf8"),t=JSON.parse(e);if(Array.isArray(t.engines)&&t.engines.length>0&&t.engines.every((r)=>typeof r==="string")){let r=t.engines.filter((i)=>ve[i]),n=t.engines.filter((i)=>!ve[i]);if(n.length>0)process.stderr.write(`[greedysearch] Warning: ignoring unknown engine(s) in ${U}: ${n.join(", ")}
|
|
359
359
|
[greedysearch] Available engines: ${Object.keys(ve).join(", ")}
|
|
360
360
|
`);if(r.length>0)return r;process.stderr.write(`[greedysearch] Warning: no valid engines in ${U}, falling back to defaults: ${ke.join(", ")}
|
|
361
|
-
`)}}}catch{}return ke}function
|
|
362
|
-
`,"utf8")}catch{}}function
|
|
361
|
+
`)}}}catch{}return ke}function zi(){try{if(!Ee(Ce))bi(Ce,{recursive:!0});if(!Ee(U))vi(U,JSON.stringify({engines:ke,synthesizer:nt},null,2)+`
|
|
362
|
+
`,"utf8")}catch{}}function Zi(){try{if(Ee(U)){let e=lr(U,"utf8"),t=JSON.parse(e);if(typeof t.synthesizer==="string"){let r=t.synthesizer.toLowerCase();if(it.includes(r))return r;process.stderr.write(`[greedysearch] Warning: unknown synthesizer "${t.synthesizer}" in ${U}
|
|
363
363
|
[greedysearch] Available synthesizers: ${it.join(", ")}
|
|
364
364
|
[greedysearch] Falling back to default: ${nt}
|
|
365
|
-
`)}}}catch{}return nt}function
|
|
365
|
+
`)}}}catch{}return nt}function Xi(){if(typeof globalThis.DOMMatrix>"u")globalThis.DOMMatrix=class{constructor(e=void 0){}multiplySelf(){return this}preMultiplySelf(){return this}translateSelf(){return this}scaleSelf(){return this}rotateSelf(){return this}};if(typeof globalThis.ImageData>"u")globalThis.ImageData=class{constructor(e=void 0,t=0,r=0){this.data=e,this.width=t,this.height=r}};if(typeof globalThis.Path2D>"u")globalThis.Path2D=class{constructor(e=void 0){}}}async function ea(){Xi();let e=await import("pdf-parse"),t=e.PDFParse??e.default;if(!t)throw Error("pdf-parse did not export PDFParse");return t}async function ta(e,t){try{let r=new(await ea())({data:new Uint8Array(e)});await r.load();let n=await r.getText(),i=n.text?.trim();if(!i)return null;return{title:new URL(t).pathname.split("/").pop()||"Document.pdf",content:`## PDF Content (${n.total} pages)
|
|
366
366
|
|
|
367
|
-
${i}`,pages:n.total}}catch(r){return{error:r.message||String(r)}}}function
|
|
368
|
-
`)}}if(
|
|
369
|
-
`);let i=await
|
|
370
|
-
`)}let n=await
|
|
371
|
-
`),await
|
|
367
|
+
${i}`,pages:n.total}}catch(r){return{error:r.message||String(r)}}}function gr(e="",t=240){let r=String(e).replaceAll(/\s+/g," ").trim();if(r.length<=t)return r;let n=r.slice(0,t),i=n.lastIndexOf(" ");return i>0?`${n.slice(0,i)}...`:`${n}...`}async function ra(e,t,r=8000){let n=Date.now();try{let i=(await R(["evalraw",e,"Page.getFrameTree","{}"]).then((p)=>JSON.parse(p)).catch(()=>null))?.frameTree?.frame?.id||void 0,a=await R(["evalraw",e,"Network.loadNetworkResource",JSON.stringify({frameId:i,url:t,options:{disableCache:!0,includeCredentials:!1}})],20000),o=JSON.parse(a).resource;if(!o?.success||!o.httpStatusCode)return{url:t,error:o?.netErrorName||o?.netError||"loadNetworkResource failed",source:"chrome",duration:Date.now()-n,needsFallback:!0};let s="";if(o.stream)try{let p=await R(["evalraw",e,"IO.read",JSON.stringify({handle:o.stream})],1e4);s=JSON.parse(p).data||"",await R(["evalraw",e,"IO.close",JSON.stringify({handle:o.stream})]).catch(()=>{})}catch{}if(!s||s.length<100)return{url:t,error:"Empty response body from Network.loadNetworkResource",source:"chrome",duration:Date.now()-n,needsFallback:!0};let l=cr(o.httpStatusCode,s,t,t);if(l.blocked)return{url:t,status:o.httpStatusCode,error:`Blocked: ${l.reason}`,source:"chrome",duration:Date.now()-n,needsBrowser:!0};let c=await ur(s,t),u=dr(c);if(!u.ok)return{url:t,status:o.httpStatusCode,error:`Low quality: ${u.reason}`,source:"chrome",duration:Date.now()-n,needsBrowser:!0};let d=re(c.markdown,r);return{url:t,finalUrl:t,status:o.httpStatusCode,contentType:"text/markdown",lastModified:"",publishedTime:c.publishedTime||"",byline:c.byline||"",siteName:c.siteName||"",lang:c.lang||"",title:c.title||t,snippet:c.excerpt,content:d,contentChars:d.length,source:"chrome",duration:Date.now()-n}}catch(i){return{url:t,error:i.message,source:"chrome",duration:Date.now()-n,needsFallback:!0}}}function mr(e){try{return new URL(e).pathname.toLowerCase().endsWith(".pdf")}catch{return!1}}async function na(e,t=8000){let r=ze(e);if(r.blocked)return{url:e,finalUrl:e,status:403,error:`Blocked: ${r.reason}`,source:"pdf-http"};let n=new AbortController,i=setTimeout(()=>n.abort(),20000),a=Date.now();try{let o=await fetch(e,{method:"GET",redirect:"follow",signal:n.signal,headers:Di({accept:"application/pdf,application/octet-stream;q=0.9,*/*;q=0.5"})});clearTimeout(i);let s=o.headers.get("content-type")||"",l=o.url||e,c=Number.parseInt(o.headers.get("content-length")||"0",10);if(o.status>=400)return{url:e,finalUrl:l,status:o.status,error:`HTTP ${o.status}`,source:"pdf-http",duration:Date.now()-a};if(!s.toLowerCase().includes("application/pdf")&&!mr(l))return null;if(c>31457280)return{url:e,finalUrl:l,status:o.status,error:`PDF too large: ${c} bytes`,source:"pdf-http",duration:Date.now()-a};let u=Buffer.from(await o.arrayBuffer()),d=await ta(u,l);if(!d||d.error)return{url:e,finalUrl:l,status:o.status,error:d?.error||"PDF text extraction failed",source:"pdf-http",duration:Date.now()-a};let p=re(d.content,t);return{url:e,finalUrl:l,status:o.status,contentType:"application/pdf",lastModified:o.headers.get("last-modified")||"",title:d.title,snippet:gr(p,320),content:p,contentChars:p.length,pages:d.pages,source:"pdf-http",duration:Date.now()-a}}catch(o){return clearTimeout(i),{url:e,finalUrl:e,error:o.message||String(o),source:"pdf-http",duration:Date.now()-a}}}async function yr(e,t=8000){let r=Date.now();if(mr(e)){let i=await na(e,t);if(i?.content||i?.status===403)return i}if(Ze(e)){let i=Ze(e);if(i&&(i.type==="root"||i.type==="tree"||i.type==="blob"&&!i.path?.includes("."))){let a=await xi(e);if(a.ok){let o=re(a.content,t);return{url:e,finalUrl:e,status:200,contentType:"text/markdown",lastModified:"",title:a.title,snippet:o.slice(0,320),content:o,contentChars:o.length,source:"github-api",...a.tree&&{tree:a.tree},duration:Date.now()-r}}process.stderr.write(`[greedysearch] GitHub API fetch failed, trying HTTP: ${a.error}
|
|
368
|
+
`)}}if(Ii(e)?.type==="post"){process.stderr.write(`[greedysearch] Using Reddit JSON API for: ${e.slice(0,60)}...
|
|
369
|
+
`);let i=await ji(e,t);if(i.ok){let a=re(i.markdown,t);return{url:e,finalUrl:i.finalUrl,status:i.status,contentType:"text/markdown",lastModified:i.lastModified||"",publishedTime:i.publishedTime||"",byline:i.byline||"",siteName:i.siteName||"",lang:i.lang||"",title:i.title,snippet:i.excerpt,content:a,contentChars:a.length,source:"reddit-api",duration:Date.now()-r}}process.stderr.write(`[greedysearch] Reddit API fetch failed, falling back to HTTP: ${i.error}
|
|
370
|
+
`)}let n=await Ci(e,{timeoutMs:1e4});if(n.ok){let i=re(n.markdown,t);return{url:e,finalUrl:n.finalUrl,status:n.status,contentType:"text/markdown",lastModified:n.lastModified||"",publishedTime:n.publishedTime||"",byline:n.byline||"",siteName:n.siteName||"",lang:n.lang||"",title:n.title,snippet:n.excerpt,content:i,contentChars:i.length,source:"http",duration:Date.now()-r}}if(n.needsBrowser)try{let i=await ft();try{let a=await ra(i,e,t);if(a.content&&a.content.length>100)return a}finally{await pt(i)}}catch{}return process.stderr.write(`[greedysearch] HTTP failed for ${e.slice(0,60)}, trying browser...
|
|
371
|
+
`),await ia(e,t)}async function wr(e,t=4000,r=200){let n=Date.now()+t;while(Date.now()<n){try{if((await R(["eval",e,'document.readyState === "complete" && !!document.body && document.body.innerText.length > 500'])).trim()==="true")return}catch{}await new Promise((i)=>setTimeout(i,r))}}async function ia(e,t=8000){let r=Date.now(),n;try{n=await ft()}catch(i){return{url:e,title:"",content:null,snippet:"",contentChars:0,error:`openNewTab failed: ${i.message}`,source:"browser",duration:Date.now()-r}}try{await R(["nav",n,e],30000),await wr(n);let i=await R(["eval",n,String.raw`
|
|
372
372
|
(function(){
|
|
373
373
|
var el = document.querySelector('article, [role="main"], main, .post-content, .article-body, #content, .content');
|
|
374
374
|
var text = (el || document.body).innerText;
|
|
@@ -378,58 +378,58 @@ ${i}`,pages:n.total}}catch(r){return{error:r.message||String(r)}}}function hr(t=
|
|
|
378
378
|
url: location.href
|
|
379
379
|
});
|
|
380
380
|
})()
|
|
381
|
-
`]),a=JSON.parse(i),o=re(a.content,
|
|
382
|
-
`);let o=Array(i.length),s=0,l=0;async function c(){while(!0){let f=s++;if(f>=i.length)return;let m=i[f],
|
|
383
|
-
`);let
|
|
384
|
-
`);else if(
|
|
381
|
+
`]),a=JSON.parse(i),o=re(a.content,t);return{url:e,finalUrl:a.url||e,status:200,contentType:"text/plain",lastModified:"",title:a.title,snippet:gr(o,320),content:o,contentChars:o.length,source:"browser",duration:Date.now()-r}}catch(i){return{url:e,title:"",content:null,snippet:"",contentChars:0,error:i.message,source:"browser",duration:Date.now()-r}}finally{await pt(n)}}async function aa(e,t=5,r=8000,n=at){let i=e.slice(0,t);if(i.length===0)return[];let a=Math.min(i.length,Math.max(1,Number.parseInt(String(n),10)||at));process.stderr.write(`[greedysearch] Fetching content from ${i.length} sources via HTTP (concurrency ${a})...
|
|
382
|
+
`);let o=Array(i.length),s=0,l=0;async function c(){while(!0){let f=s++;if(f>=i.length)return;let m=i[f],h=m.canonicalUrl||m.url;process.stderr.write(`[greedysearch] [${f+1}/${i.length}] Fetching: ${h.slice(0,60)}...
|
|
383
|
+
`);let g=await yr(h,r).catch((y)=>({url:h,title:"",content:null,snippet:"",contentChars:0,error:y.message,source:"error",duration:0}));if(o[f]={id:m.id,...g},g.content&&g.content.length>100)process.stderr.write(`[greedysearch] ✓ ${g.source}: ${g.content.length} chars
|
|
384
|
+
`);else if(g.error)process.stderr.write(`[greedysearch] ✗ ${g.error.slice(0,80)}
|
|
385
385
|
`);l+=1,process.stderr.write(`PROGRESS:fetch:${l}/${i.length}
|
|
386
386
|
`)}}await Promise.all(Array.from({length:a},()=>c()));let u=o.filter((f)=>f.content&&f.content.length>100),d=o.filter((f)=>f.source==="http").length,p=o.filter((f)=>f.source==="browser").length;return process.stderr.write(`[greedysearch] Fetched ${u.length}/${o.length} sources (HTTP: ${d}, Browser: ${p})
|
|
387
|
-
`),o}async function
|
|
387
|
+
`),o}async function oa(e){let t=await ft();try{await R(["nav",t,e],30000),await wr(t);let r=await R(["eval",t,String.raw`
|
|
388
388
|
(function(){
|
|
389
389
|
var el = document.querySelector('article, [role="main"], main, .post-content, .article-body, #content, .content');
|
|
390
390
|
var text = (el || document.body).innerText;
|
|
391
391
|
return text.replace(/\s+/g, ' ').trim();
|
|
392
392
|
})()
|
|
393
|
-
`]);return{url:
|
|
393
|
+
`]);return{url:e,content:r}}catch(r){return{url:e,content:null,error:r.message}}finally{await pt(t)}}var sa,la,me=null,br="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",ht,gt,vr,$r,_r,Er,Sr,Rr,mt,Dr,Kt,Ar,Cr,Vt,ca,O,ua,da,fa,pa,ha,ga,ma,ya,wa,Se,L,Re,Xe="gemini",et,we,ba,va,$a,zt,_a,N,Ea,Sa,Ra,Da,Aa,Ca,ka,Oa,Na,De,M,Ae,tt="gemini",rt,be,Pa,xa,Ia,ja,Ta,La,Ma,R,Zt,Ua,P,Ga,Fa,Ha,qa,Ja,Ya,Wa,Ba,Qa,Ce,U,ke,nt="gemini",it,ve,Ka,Va,at,za=dt(()=>{sa=ni(import.meta.url),la=ii(import.meta.url),ht={"user-agent":br,accept:"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8","accept-language":"en-US,en;q=0.9","accept-encoding":"gzip, deflate, br","cache-control":"no-cache",pragma:"no-cache","sec-ch-ua":'"Chromium";v="122", "Not(A:Brand";v="24", "Google Chrome";v="122"',"sec-ch-ua-mobile":"?0","sec-ch-ua-platform":'"Windows"',"sec-fetch-dest":"document","sec-fetch-mode":"navigate","sec-fetch-site":"none","sec-fetch-user":"?1","upgrade-insecure-requests":"1"},gt=[/^localhost$/i,/^127\.\d+\.\d+\.\d+$/,/^0\.0\.0\.0$/,/^\[::1\]$/,/^10\./,/^172\.(1[6-9]|2\d|3[01])\./,/^192\.168\./,/^169\.254\./,/^fc00:/i,/^fe80:/i,/\.local$/i,/\.internal$/i,/\.localhost$/i],vr=/^::ffff:(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/i,$r=/^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/i,_r=/^(?:0x[0-9a-f]+|0[0-7]*|[1-9][0-9]*)$/i,Er=["accounts.google.com","login.microsoftonline.com","login.live.com","auth0.com","okta.com","auth.mozilla.auth0.com","id.atlassian.com"],Sr=["login.","signin.","auth.","sso.","accounts.","idp."],Rr=["sign in to continue","log in to continue","authentication required","create an account to continue","subscribe to continue reading","members only"],mt={"user-agent":"GreedySearch/1.0",accept:"application/vnd.github+json","x-github-api-version":"2022-11-28"},Dr={"user-agent":"GreedySearch/1.0 (Research Bot)",accept:"application/json"},Kt=si(ci(import.meta.url)),Ar=li(Kt,"..","bin","cdp.mjs"),Cr=new Set(["list","snap","eval","shot","html","nav","net","click","clickxy","type","loadall","evalraw","browse","stop","--tab"]),Vt=hi().replaceAll("\\","/"),ca=Fi(process.env.GREEDY_SEARCH_PORT),O=(process.env.GREEDY_SEARCH_PROFILE_DIR||process.env.CDP_PROFILE_DIR||`${Vt}/greedysearch-chrome-profile`).replaceAll("\\","/"),ua=`${O}/DevToolsActivePort`,da=process.env.GREEDY_SEARCH_PID_FILE||`${O}/browser.pid`,fa=process.env.CDP_PAGES_CACHE||`${O}/cdp-pages.json`,pa=process.env.GREEDY_SEARCH_MODE_FILE||`${O}/browser-mode`,ha=process.env.GREEDY_SEARCH_METADATA_FILE||`${O}/browser-metadata.json`,ga=process.env.GREEDY_SEARCH_LAUNCH_LOCK_FILE||`${O}/browser-launch.lock`,ma=process.env.GREEDY_SEARCH_ACTIVITY_FILE||`${O}/browser-last-activity`,ya=(process.env.CDP_SOCKET_DIR||`${O}/cdp-sockets`).replaceAll("\\","/"),wa=`${O}/visible-recovery.jsonl`,Se=Gt(pi(),".dm"),L=Gt(Se,"greedyconfig"),Re=["perplexity","google","chatgpt","gemini"],qi(),et=["gemini","chatgpt"],we={perplexity:"perplexity.mjs",p:"perplexity.mjs",bing:"bing-copilot.mjs",b:"bing-copilot.mjs",google:"google-ai.mjs",g:"google-ai.mjs",gemini:"gemini.mjs",gem:"gemini.mjs",chatgpt:"chatgpt.mjs",gpt:"chatgpt.mjs","semantic-scholar":"semantic-scholar.mjs",semanticscholar:"semantic-scholar.mjs",s2:"semantic-scholar.mjs",logically:"logically.mjs",log:"logically.mjs"},ba=Hi(),va=Ji(),$a=Math.max(1,Number.parseInt(process.env.GREEDY_FETCH_CONCURRENCY||"5",10)||5),process.env.CDP_PROFILE_DIR=O,zt=wi().replaceAll("\\","/"),_a=Yi(process.env.GREEDY_SEARCH_PORT),N=(process.env.GREEDY_SEARCH_PROFILE_DIR||process.env.CDP_PROFILE_DIR||`${zt}/greedysearch-chrome-profile`).replaceAll("\\","/"),Ea=`${N}/DevToolsActivePort`,Sa=process.env.GREEDY_SEARCH_PID_FILE||`${N}/browser.pid`,Ra=process.env.CDP_PAGES_CACHE||`${N}/cdp-pages.json`,Da=process.env.GREEDY_SEARCH_MODE_FILE||`${N}/browser-mode`,Aa=process.env.GREEDY_SEARCH_METADATA_FILE||`${N}/browser-metadata.json`,Ca=process.env.GREEDY_SEARCH_LAUNCH_LOCK_FILE||`${N}/browser-launch.lock`,ka=process.env.GREEDY_SEARCH_ACTIVITY_FILE||`${N}/browser-last-activity`,Oa=(process.env.CDP_SOCKET_DIR||`${N}/cdp-sockets`).replaceAll("\\","/"),Na=`${N}/visible-recovery.jsonl`,De=Ft(yi(),".dm"),M=Ft(De,"greedyconfig"),Ae=["perplexity","google","chatgpt","gemini"],Bi(),rt=["gemini","chatgpt"],be={perplexity:"perplexity.mjs",p:"perplexity.mjs",bing:"bing-copilot.mjs",b:"bing-copilot.mjs",google:"google-ai.mjs",g:"google-ai.mjs",gemini:"gemini.mjs",gem:"gemini.mjs",chatgpt:"chatgpt.mjs",gpt:"chatgpt.mjs","semantic-scholar":"semantic-scholar.mjs",semanticscholar:"semantic-scholar.mjs",s2:"semantic-scholar.mjs",logically:"logically.mjs",log:"logically.mjs"},Pa=Wi(),xa=Qi(),Ia=Math.max(1,Number.parseInt(process.env.GREEDY_FETCH_CONCURRENCY||"5",10)||5),process.env.CDP_PROFILE_DIR=N,ja=Number.parseInt(process.env.GREEDY_SEARCH_IDLE_TIMEOUT_MINUTES||"5",10)||5,Ta=Gi(import.meta.url),La=Number.parseInt(process.env.GREEDY_SEARCH_IDLE_TIMEOUT_MINUTES||"5",10)||5,Ma=Number.parseInt(process.env.GREEDY_SEARCH_VISIBLE_IDLE_TIMEOUT_MINUTES||"60",10)||60,R=pr,Zt=_i().replaceAll("\\","/"),Ua=Ki(process.env.GREEDY_SEARCH_PORT),P=(process.env.GREEDY_SEARCH_PROFILE_DIR||process.env.CDP_PROFILE_DIR||`${Zt}/greedysearch-chrome-profile`).replaceAll("\\","/"),Ga=`${P}/DevToolsActivePort`,Fa=process.env.GREEDY_SEARCH_PID_FILE||`${P}/browser.pid`,Ha=process.env.CDP_PAGES_CACHE||`${P}/cdp-pages.json`,qa=process.env.GREEDY_SEARCH_MODE_FILE||`${P}/browser-mode`,Ja=process.env.GREEDY_SEARCH_METADATA_FILE||`${P}/browser-metadata.json`,Ya=process.env.GREEDY_SEARCH_LAUNCH_LOCK_FILE||`${P}/browser-launch.lock`,Wa=process.env.GREEDY_SEARCH_ACTIVITY_FILE||`${P}/browser-last-activity`,Ba=(process.env.CDP_SOCKET_DIR||`${P}/cdp-sockets`).replaceAll("\\","/"),Qa=`${P}/visible-recovery.jsonl`,Ce=Ht($i(),".dm"),U=Ht(Ce,"greedyconfig"),ke=["perplexity","google","chatgpt","gemini"],zi(),it=["gemini","chatgpt"],ve={perplexity:"perplexity.mjs",p:"perplexity.mjs",bing:"bing-copilot.mjs",b:"bing-copilot.mjs",google:"google-ai.mjs",g:"google-ai.mjs",gemini:"gemini.mjs",gem:"gemini.mjs",chatgpt:"chatgpt.mjs",gpt:"chatgpt.mjs","semantic-scholar":"semantic-scholar.mjs",semanticscholar:"semantic-scholar.mjs",s2:"semantic-scholar.mjs",logically:"logically.mjs",log:"logically.mjs"},Ka=Vi(),Va=Zi(),at=Math.max(1,Number.parseInt(process.env.GREEDY_FETCH_CONCURRENCY||"5",10)||5),process.env.CDP_PROFILE_DIR=P}),Za={};ut(Za,{parseGitHubUrl:()=>kr,fetchGitHubContent:()=>to});function kr(e){try{let t=new URL(e);if(!(t.hostname==="github.com"||t.hostname.endsWith(".github.com")))return null;let r=t.pathname.split("/").filter(Boolean);if(r.length<2)return null;let[n,i]=r;if(r.length===2)return{owner:n,repo:i,type:"root"};if(r.length>=4&&(r[2]==="blob"||r[2]==="tree")){let a=r[2],o=r[3],s=r.slice(4).join("/");return{owner:n,repo:i,type:a,ref:o,path:s}}return null}catch{return null}}async function I(e,t=1e4){let r=new AbortController,n=setTimeout(()=>r.abort(),t);try{let i=await fetch(`https://api.github.com${e}`,{headers:yt,signal:r.signal});if(clearTimeout(n),!i.ok)throw Error(`GitHub API ${i.status}: ${e}`);return await i.json()}catch(i){throw clearTimeout(n),i}}async function Xa(e,t){try{let r=await I(`/repos/${e}/${t}/readme`);if(r.content&&r.encoding==="base64")return Buffer.from(r.content,"base64").toString("utf8");return""}catch{return""}}async function Xt(e,t,r="HEAD",n="",i){try{let a;if(r==="HEAD")if(i)a=await I(`/repos/${e}/${t}/git/ref/heads/${i}`).catch(()=>null);else a=await Promise.any([I(`/repos/${e}/${t}/git/ref/heads/main`),I(`/repos/${e}/${t}/git/ref/heads/master`)]).catch(()=>null);else a=await I(`/repos/${e}/${t}/git/ref/heads/${r}`).catch(()=>I(`/repos/${e}/${t}/git/ref/heads/master`).catch(()=>null));if(!a?.object?.sha)return[];let o=(await I(`/repos/${e}/${t}/git/commits/${a.object.sha}`)).tree.sha,s=(await I(`/repos/${e}/${t}/git/trees/${o}`)).tree||[];if(n)s=s.filter((l)=>l.path.startsWith(n));return s.slice(0,50).map((l)=>({path:l.path,type:l.type==="tree"?"dir":"file",size:l.size}))}catch{return[]}}async function eo(e,t,r,n,i=1e4,a){let o=async(l)=>{let c=new AbortController,u=setTimeout(()=>c.abort(),i);try{let d=await fetch(l,{headers:{"user-agent":yt["user-agent"]},signal:c.signal});if(clearTimeout(u),d.ok)return await d.text();throw Error("not ok")}catch{throw clearTimeout(u),Error("failed")}};if(!r||r==="HEAD"){if(a)try{return await o(`https://raw.githubusercontent.com/${e}/${t}/${a}/${n}`)}catch{return null}try{return await Promise.any([o(`https://raw.githubusercontent.com/${e}/${t}/main/${n}`),o(`https://raw.githubusercontent.com/${e}/${t}/master/${n}`)])}catch{return null}}let s=[`https://raw.githubusercontent.com/${e}/${t}/${r}/${n}`,`https://raw.githubusercontent.com/${e}/${t}/master/${n}`];for(let l of s)try{return await o(l)}catch{}return null}async function to(e){let t=kr(e);if(!t)return{ok:!1,error:"Not a valid GitHub URL"};let{owner:r,repo:n,type:i,ref:a,path:o}=t;try{if(i==="root"||i==="tree"&&!o){let s=await I(`/repos/${r}/${n}`),[l,c]=await Promise.allSettled([Xa(r,n),Xt(r,n,a||"HEAD","",s.default_branch)]),u=l.status==="fulfilled"?l.value:"",d=c.status==="fulfilled"?c.value:[],p=s?.description?`
|
|
394
394
|
|
|
395
|
-
> ${s.description}`:"",f=s?.stargazers_count==null?"":` ⭐ ${s.stargazers_count}`,m=s?.language?` · ${s.language}`:"",
|
|
395
|
+
> ${s.description}`:"",f=s?.stargazers_count==null?"":` ⭐ ${s.stargazers_count}`,m=s?.language?` · ${s.language}`:"",h=`# ${r}/${n}${f}${m}${p}
|
|
396
396
|
|
|
397
|
-
`;if(u)
|
|
397
|
+
`;if(u)h+=u.slice(0,6000);else h+=`[No README found]
|
|
398
398
|
|
|
399
399
|
Files:
|
|
400
|
-
${d.map((
|
|
401
|
-
`)}`;return{ok:!0,title:`${r}/${n}`,content:
|
|
400
|
+
${d.map((g)=>` ${g.type==="dir"?"\uD83D\uDCC1":"\uD83D\uDCC4"} ${g.path}`).join(`
|
|
401
|
+
`)}`;return{ok:!0,title:`${r}/${n}`,content:h,tree:d.slice(0,30)}}if(i==="blob"&&o){let s;if(!a||a==="HEAD")try{s=(await I(`/repos/${r}/${n}`)).default_branch}catch{s=void 0}let l=await eo(r,n,a,o,1e4,s);if(l===null)return{ok:!1,error:`File not found: ${o}`};return{ok:!0,title:`${r}/${n}: ${o}`,content:l}}if(i==="tree"&&o){let s;if(!a||a==="HEAD")try{s=(await I(`/repos/${r}/${n}`)).default_branch}catch{s=void 0}let l=await Xt(r,n,a||"HEAD",o,s),c=l.map((u)=>` ${u.type==="dir"?"\uD83D\uDCC1":"\uD83D\uDCC4"} ${u.path}`).join(`
|
|
402
402
|
`);return{ok:!0,title:`${r}/${n}/${o}`,content:`[Directory: ${o}]
|
|
403
403
|
|
|
404
404
|
Files:
|
|
405
|
-
${c}`,tree:l}}return{ok:!1,error:"Unsupported GitHub URL type"}}catch(s){return{ok:!1,error:s.message}}}var yt,
|
|
406
|
-
[greedysearch] Available engines: ${Object.keys(
|
|
405
|
+
${c}`,tree:l}}return{ok:!1,error:"Unsupported GitHub URL type"}}catch(s){return{ok:!1,error:s.message}}}var yt,Cc=dt(()=>{yt={"user-agent":"GreedySearch/1.0",accept:"application/vnd.github+json","x-github-api-version":"2022-11-28"}});function no(e=process.env,t=process.execPath){let r=e.GREEDY_SEARCH_NODE||e.NODE_BINARY||e.NODE;if(r?.trim())return r.trim();let n=ro(t||"").toLowerCase();if(n==="node"||n==="node.exe")return t;return"node"}var ao=["fbclid","gclid","ref","ref_src","ref_url","source","utm_campaign","utm_content","utm_medium","utm_source","utm_term"];function z(e="",t=240){let r=String(e).replaceAll(/\s+/g," ").trim();if(r.length<=t)return r;let n=r.slice(0,t),i=n.lastIndexOf(" ");return i>0?`${n.slice(0,i)}...`:`${n}...`}function ot(e){if(!e)return null;try{let t=new URL(e);if(!["http:","https:"].includes(t.protocol))return null;if(t.hash="",t.hostname=t.hostname.toLowerCase(),t.protocol==="https:"&&t.port==="443"||t.protocol==="http:"&&t.port==="80")t.port="";for(let i of[...t.searchParams.keys()]){let a=i.toLowerCase();if(ao.includes(a)||a.startsWith("utm_"))t.searchParams.delete(i)}t.searchParams.sort();let r=t.pathname.replace(/\/{1,10}$/,"")||"/";t.pathname=r;let n=t.toString();return r==="/"?n.replace(/\/$/,""):n}catch{return null}}function uo(e,t=9222){let r=Number.parseInt(String(e??""),10);return Number.isInteger(r)&&r>1024&&r<65535?r:t}var fo=co().replaceAll("\\","/"),Lc=uo(process.env.GREEDY_SEARCH_PORT),j=(process.env.GREEDY_SEARCH_PROFILE_DIR||process.env.CDP_PROFILE_DIR||`${fo}/greedysearch-chrome-profile`).replaceAll("\\","/"),Mc=`${j}/DevToolsActivePort`,Uc=process.env.GREEDY_SEARCH_PID_FILE||`${j}/browser.pid`,Gc=process.env.CDP_PAGES_CACHE||`${j}/cdp-pages.json`,Fc=process.env.GREEDY_SEARCH_MODE_FILE||`${j}/browser-mode`,Hc=process.env.GREEDY_SEARCH_METADATA_FILE||`${j}/browser-metadata.json`,qc=process.env.GREEDY_SEARCH_LAUNCH_LOCK_FILE||`${j}/browser-launch.lock`,Jc=process.env.GREEDY_SEARCH_ACTIVITY_FILE||`${j}/browser-last-activity`,Yc=(process.env.CDP_SOCKET_DIR||`${j}/cdp-sockets`).replaceAll("\\","/"),Wc=`${j}/visible-recovery.jsonl`,st=Nr(lo(),".dm"),J=Nr(st,"greedyconfig"),lt=["perplexity","google","chatgpt","gemini"],ct="gemini";function po(){try{if(Oe(J)){let e=Or(J,"utf8"),t=JSON.parse(e);if(Array.isArray(t.engines)&&t.engines.length>0&&t.engines.every((r)=>typeof r==="string")){let r=t.engines.filter((i)=>Ke[i]),n=t.engines.filter((i)=>!Ke[i]);if(n.length>0)process.stderr.write(`[greedysearch] Warning: ignoring unknown engine(s) in ${J}: ${n.join(", ")}
|
|
406
|
+
[greedysearch] Available engines: ${Object.keys(Ke).join(", ")}
|
|
407
407
|
`);if(r.length>0)return r;process.stderr.write(`[greedysearch] Warning: no valid engines in ${J}, falling back to defaults: ${lt.join(", ")}
|
|
408
|
-
`)}}}catch{}return lt}function
|
|
409
|
-
`,"utf8")}catch{}}
|
|
408
|
+
`)}}}catch{}return lt}function ho(){try{if(!Oe(st))oo(st,{recursive:!0});if(!Oe(J))so(J,JSON.stringify({engines:lt,synthesizer:ct},null,2)+`
|
|
409
|
+
`,"utf8")}catch{}}ho();var Ne=["gemini","chatgpt"];function go(){try{if(Oe(J)){let e=Or(J,"utf8"),t=JSON.parse(e);if(typeof t.synthesizer==="string"){let r=t.synthesizer.toLowerCase();if(Ne.includes(r))return r;process.stderr.write(`[greedysearch] Warning: unknown synthesizer "${t.synthesizer}" in ${J}
|
|
410
410
|
[greedysearch] Available synthesizers: ${Ne.join(", ")}
|
|
411
411
|
[greedysearch] Falling back to default: ${ct}
|
|
412
|
-
`)}}}catch{}return ct}var
|
|
413
|
-
`)
|
|
414
|
-
`||n==="\r"){if(!r)
|
|
415
|
-
`)}async function Tr({query:
|
|
416
|
-
`)}function Mr(t
|
|
417
|
-
`)}var
|
|
418
|
-
`);try{let r=new Set(
|
|
412
|
+
`)}}}catch{}return ct}var Ke={perplexity:"perplexity.mjs",p:"perplexity.mjs",bing:"bing-copilot.mjs",b:"bing-copilot.mjs",google:"google-ai.mjs",g:"google-ai.mjs",gemini:"gemini.mjs",gem:"gemini.mjs",chatgpt:"chatgpt.mjs",gpt:"chatgpt.mjs","semantic-scholar":"semantic-scholar.mjs",semanticscholar:"semantic-scholar.mjs",s2:"semantic-scholar.mjs",logically:"logically.mjs",log:"logically.mjs"},Pr=po();var Bc=go(),Qc=Math.max(1,Number.parseInt(process.env.GREEDY_FETCH_CONCURRENCY||"5",10)||5);process.env.CDP_PROFILE_DIR=j;function mo(e){let t="",r=!1,n=!1;for(let i of String(e)){if(n){t+=i,n=!1;continue}if(i==="\\"){t+=i,n=!0;continue}if(i==='"'){r=!r,t+=i;continue}if(r&&i===`
|
|
413
|
+
`)t+="\\n";else if(r&&i==="\r")t+="\\r";else if(r&&i==="\t")t+="\\t";else t+=i}return t}function yo(e){if(!e)return null;let t=String(e).trim(),r=t.indexOf("BEGIN_JSON"),n=t.indexOf("END_JSON");if(r!==-1&&n!==-1&&r<n)t=t.slice(r+10,n).trim();else{let s=t.indexOf("{");if(s>0)t=t.slice(s)}let i=[t,t.replace(/^```json\s*/i,"").replace(/^```\s*/i,"").replace(/```$/i,"").trim()],a=t.indexOf("{"),o=t.lastIndexOf("}");if(a!==-1&&o!==-1&&a<o)i.push(t.slice(a,o+1));for(let s of[...i]){let l=mo(s);if(l!==s)i.push(l)}for(let s of i)try{return JSON.parse(s)}catch{}return null}function $o(e){return vo(new URL(".",e)).replace(/^\/([A-Z]:)/,"$1")}function _o(e,t=xr){let r=new Set;for(let n of e.filter(Boolean)){if(r.has(n))continue;if(r.add(n),t(n))return n}return e.filter(Boolean).at(-1)}function Eo(e,{moduleDir:t,entrypoint:r=process.argv[1],env:n=process.env,exists:i=xr}={}){let a=r?bo(r):null,o=n.GREEDY_SEARCH_EXTENSION_DIR?.trim()||null;return _o([t?ye(t,"..","..","extractors",e):null,t?ye(t,"..","extractors",e):null,o?ye(o,"extractors",e):null,a?ye(a,"..","extractors",e):null],i)}var So=$o(import.meta.url),Ro={gemini:"gemini.mjs",chatgpt:"chatgpt.mjs"};function Do(e="gemini"){let t=String(e||"gemini").toLowerCase();if(t==="gem")return"gemini";if(t==="gpt")return"chatgpt";return t}async function Ao(e,t,{tabPrefix:r=null,timeoutMs:n=180000,visible:i=null}={}){let a=Do(e),o=Ro[a];if(!o||!Ne.includes(a))throw Error(`Unsupported synthesizer "${e}". Supported: ${Ne.join(", ")}`);return new Promise((s,l)=>{let c=Eo(o,{moduleDir:So}),u=r?["--tab",String(r)]:[],d={...process.env,CDP_PROFILE_DIR:j};if(i!==!0)delete d.GREEDY_SEARCH_VISIBLE,delete d.GREEDY_SEARCH_ALWAYS_VISIBLE;else d.GREEDY_SEARCH_VISIBLE="1",d.GREEDY_SEARCH_ALWAYS_VISIBLE="1";let p=wo(no(),[c,"--stdin",...u],{stdio:["pipe","pipe","pipe"],env:d});p.stdin.write(t),p.stdin.end();let f="",m="";p.stdout.on("data",(g)=>f+=g),p.stderr.on("data",(g)=>m+=g);let h=setTimeout(()=>{p.kill(),l(Error(`${a} prompt timed out after ${n/1000}s`))},n);p.on("close",(g)=>{if(clearTimeout(h),g!==0){l(Error(m.trim()||`${a} extractor failed`));return}try{s(JSON.parse(f.trim()))}catch{l(Error(`bad JSON from ${a}: ${f.slice(0,100)}`))}})})}async function Co(e,t={}){return Ao("gemini",e,t)}ar();za();var No=Oo(new URL(".",import.meta.url)).replace(/^\/([A-Z]:)/,"$1"),tu=ko(No,"..","..","bin","search.mjs");var Po=Pr.join("|"),ru=new RegExp(`^\\[(${Po})\\]`);var xo=io(new URL(".",import.meta.url)).replace(/^\/([A-Z]:)/,"$1"),nu=_(xo,"..","..","bin","search.mjs"),Io=_(process.cwd(),".pi","greedysearch-research");function jo(e){return String(e||"research").toLowerCase().replaceAll(/[^a-z0-9]+/g,"-").replaceAll(/^-|-$/g,"").slice(0,60)||"research"}function se(e,t=1/0){let r=new Set,n=[];for(let i of e||[]){let a=z(String(i||""),1000);if(!a||r.has(a))continue;if(r.add(a),n.push(a),n.length>=t)break}return n}async function To(...e){let{writeSourcesToFiles:t}=await Promise.resolve().then(()=>(ar(),rr));return t(...e)}function Ir(e){return Mo(Lo(String(e)))}function Lo(e){let t="",r=0;while(r<e.length){let n=e.indexOf("[",r);if(n===-1){t+=e.slice(r);break}let i=e.indexOf("]",n+1);if(i===-1||e[i+1]!=="("||i===n+1){t+=e.slice(r,n+1),r=n+1;continue}let a=e.indexOf(")",i+2);if(a===-1){t+=e.slice(r,n+1),r=n+1;continue}let o=e.slice(i+2,a).trimStart();if(!o.startsWith("http://")&&!o.startsWith("https://")){t+=e.slice(r,n+1),r=n+1;continue}t+=e.slice(r,n),t+=e.slice(n+1,i),r=a+1}return t}function Mo(e){let t="",r=!1;for(let n of e)if(n===" "||n==="\t"||n===`
|
|
414
|
+
`||n==="\r"){if(!r)t+=" ";r=!0}else t+=n,r=!1;return t.trim()}function er(e){return new Set(String(e).toLowerCase().normalize("NFD").replaceAll(/[\u0300-\u036f]/g,"").split(/[^\w]+/).filter((t)=>t.length>1))}function jr(e,t){let r=er(e),n=er(t),i=new Set([...r,...n]).size;if(i===0)return 1;let a=0;for(let o of r)if(n.has(o))a++;return a/i}function Pe(e){return ot(e?.finalUrl||e?.canonicalUrl||e?.url||"")||e?.id||""}function Uo(e,t){let r=Array.isArray(e?.extractions)?e.extractions:[],n=new Map,i=new Map;for(let a of t||[]){if(a?.id)i.set(String(a.id),a);let o=Pe(a);if(o)n.set(o,a)}return r.map((a)=>{let o=i.get(String(a?.sourceId||""))||n.get(ot(a?.url||"")||""),s=String(a?.sourceId||o?.id||""),l=ot(a?.url||o?.finalUrl||o?.url||""),c=Array.isArray(a?.answers)?a.answers.map((u)=>({id:String(u?.id||""),evidence:z(u?.evidence||"",500),sourceIds:[s].filter(Boolean)})).filter((u)=>u.id):[];return{sourceId:s,url:l,title:o?.title||a?.title||"",rational:z(a?.rational||"",700),evidence:z(a?.evidence||"",1600),summary:z(a?.summary||"",700),answers:c,newQuestions:se(a?.newQuestions||[],6)}}).filter((a)=>a.sourceId||a.url||a.summary||a.evidence)}function Go(e,t,r,n=new Set){let i=(t||[]).filter((o)=>o.status!=="closed").slice(0,12).map((o)=>({id:o.id,question:o.question})),a=(r||[]).filter((o)=>o?.content||o?.snippet).filter((o)=>!n.has(Pe(o))).slice(0,6).map((o,s)=>({id:o.id||`F${s+1}`,title:o.title||"",url:o.finalUrl||o.url||o.canonicalUrl||"",content:z(o.content||o.snippet||"",5000)}));return["You are doing goal-based evidence extraction for an iterative research run.","For each source, extract only information that helps answer the open questions.","Use original wording/details where useful. Do not invent answers; leave questions open if evidence is insufficient.","If a source answers one or more tracked questions, identify those question IDs explicitly.","Also propose genuinely new sub-questions discovered from the evidence.","",`Original research question: ${e}`,`Open question ledger: ${JSON.stringify(i,null,2)}`,`Fetched sources: ${JSON.stringify(a,null,2)}`,"","Respond ONLY with JSON wrapped in BEGIN_JSON / END_JSON markers:","BEGIN_JSON",JSON.stringify({extractions:[{sourceId:"S1",url:"https://example.com/source",rational:"why this source matters for the goal",evidence:"specific quoted/paraphrased evidence with numbers, dates, caveats",summary:"concise contribution to the research question",answers:[{id:"Q1",evidence:"brief evidence that closes the question"}],newQuestions:["new sub-question raised by this source"]}]},null,2),"END_JSON"].join(`
|
|
415
|
+
`)}async function Tr({query:e,questions:t,fetchedSources:r,extractedSourceKeys:n}){let i=(r||[]).filter((a)=>(a?.content||a?.snippet)&&!n.has(Pe(a)));if(i.length===0)return{evidence:[],error:""};try{let a=await Co(Go(e,t,i,n),{timeoutMs:120000}),o=Ho(a,{extractions:[]}),s=Uo(o,i);for(let l of i){let c=Pe(l);if(c)n.add(c)}return{evidence:s,error:""}}catch(a){return{evidence:[],error:a.message||String(a)}}}function Lr(e,t,r,n=[],i=[]){let a=t.flatMap((l)=>l.learnings||[]),o=t.flatMap((l)=>l.gaps||[]),s=r.slice(0,12).map((l)=>({id:l.id,title:l.title,domain:l.domain,url:l.canonicalUrl,type:l.sourceType,engines:l.engines,fetch:l.fetch?.attempted?{ok:l.fetch.ok,snippet:z(l.fetch.snippet||"",1200),publishedTime:l.fetch.publishedTime||""}:void 0}));return["You are writing the final research report for an iterative deep-research run.","Produce a thorough markdown report organized into clear sections.","","Use the learnings and source registry below. Every substantive claim MUST be backed by an [S1] citation.",'Where engines disagree, surface the conflicting claims explicitly in the "differences" array.','Include a "Key Claims" structure that maps each distinct claim to its supporting source IDs.',"","Report structure:","1. ## Summary — A 2-4 sentence executive summary of findings","2. ## Key Findings — The main findings, organized by theme or question, each with inline citations","3. ## Areas of Disagreement — Where engines or sources conflict (if any)","4. ## Limitations & Caveats — Important qualifiers, gaps, or uncertainties","",`Original research question: ${e}`,`Learnings: ${JSON.stringify(a,null,2)}`,`Known gaps/caveats: ${JSON.stringify(o,null,2)}`,`Question ledger: ${JSON.stringify(n,null,2)}`,`Goal-based extracted evidence: ${JSON.stringify(i.slice(-20),null,2)}`,`Source registry: ${JSON.stringify(s,null,2)}`,"","Respond ONLY with JSON wrapped in BEGIN_JSON / END_JSON markers:","BEGIN_JSON",JSON.stringify({answer:"markdown report with sections and inline [S1] citations",agreement:{level:"high|medium|low|mixed|conflicting",summary:"one-sentence confidence summary"},differences:["notable disagreement or conflict between sources"],caveats:["important caveat or qualification"],claims:[{claim:"specific factual statement from the research",support:"strong|moderate|weak|conflicting",sourceIds:["S1","S2"]}],recommendedSources:["S1","S2"]},null,2),"END_JSON"].join(`
|
|
416
|
+
`)}function Mr(e,t=[],r=[],n=[]){let i=t.slice(0,12).map((l)=>({id:l.id,title:l.title,domain:l.domain,url:l.canonicalUrl,type:l.sourceType,engines:l.engines})),a=n.slice(-20),o=new Set;for(let l of a)for(let c of l.answers||[])if(c?.id)o.add(c.id);let s=(r||[]).filter((l)=>l.status!=="closed").map((l)=>({id:l.id,question:l.question}));return["You are writing the final research report from goal-based extracted evidence.","Per-round learnings were not produced, but the per-source evidence extraction step succeeded.","Synthesize a thorough markdown report using ONLY the evidence below. Every substantive claim MUST be backed by an [S1] citation.","","Report structure:","1. ## Summary — A 2-4 sentence executive summary of findings","2. ## Key Findings — The main findings, organized by theme or question, each with inline citations","3. ## Limitations & Caveats — Important qualifiers, gaps, or uncertainties","",`Original research question: ${e}`,`Per-source extracted evidence: ${JSON.stringify(a,null,2)}`,`Source registry: ${JSON.stringify(i,null,2)}`,`Questions already answered by the evidence: ${JSON.stringify(Array.from(o))}`,`Questions still open after this evidence: ${JSON.stringify(s)}`,"","Respond ONLY with JSON wrapped in BEGIN_JSON / END_JSON markers:","BEGIN_JSON",JSON.stringify({answer:"markdown report with sections and inline [S1] citations",agreement:{level:"high|medium|low|mixed|conflicting",summary:"one-sentence confidence summary"},differences:["notable disagreement or conflict between sources"],caveats:["important caveat or qualification"],claims:[{claim:"specific factual statement supported by the evidence",support:"strong|moderate|weak|conflicting",sourceIds:["S1","S2"]}],recommendedSources:["S1","S2"]},null,2),"END_JSON"].join(`
|
|
417
|
+
`)}var Fo=Pr.join("|"),iu=new RegExp(`^\\[(${Fo})\\]`);function Ho(e,t={}){return yo(e?.answer||"")||t}function Ur(e,t){if(!e||!Array.isArray(t))return{cited:[],missing:[],unfetched:[],ok:!0};let r=/\b[SF](\d+)\b/g,n=new Set,i;while((i=r.exec(e))!==null)n.add(`S${i[1]}`),n.add(`F${i[1]}`);let a=new Map;for(let c of t){let u=c?.id;if(u)a.set(u,c)}let o=Array.from(n),s=[],l=[];for(let c of o){let u=a.get(c);if(!u){let d=c.match(/^(S|F)(\d+)$/);if(d){let p=parseInt(d[2],10)-1;if(p>=0&&p<t.length){let f=t[p];if(f){if(!(f.fetch?.ok||f.content&&f.content.length>100||f.contentChars&&f.contentChars>100))l.push(c);continue}}}s.push(c)}else if(!(u.fetch?.ok||u.content&&u.content.length>100||u.contentChars&&u.contentChars>100))l.push(c)}return{cited:o,missing:s,unfetched:l,ok:s.length===0}}async function qo(e,{timeoutMs:t=6000,concurrency:r=4}={}){let n=Math.max(1,Math.floor(r||1)),i=(e||[]).filter((p)=>p?.id&&(p?.canonicalUrl||p?.finalUrl||p?.url));if(i.length===0)return{reachable:[],dead:[],skipped:[],ok:!0};let a=[],o=[],s=[],l=Array(i.length),c=0;async function u(){while(!0){let p=c++;if(p>=i.length)return;let f=i[p];try{let m=f.fetch?.finalUrl||f.canonicalUrl||f.finalUrl||f.url;if(!m){l[p]={id:f.id,url:"",status:"skipped"};continue}try{let h=new URL(m);if(h.protocol!=="http:"&&h.protocol!=="https:"){l[p]={id:f.id,url:m,status:"skipped"};continue}}catch{l[p]={id:f.id,url:m,status:"skipped"};continue}try{let h=new AbortController,g=setTimeout(()=>h.abort(),t);try{let y=await fetch(m,{method:"HEAD",redirect:"follow",signal:h.signal,headers:{"User-Agent":"Mozilla/5.0 (compatible; GreedySearch/2.0; +https://github.com/apmantza/greedysearch-dm)"}});clearTimeout(g);let v=y.status>=200&&y.status<400,b=[401,403,405,429].includes(y.status),w="dead";if(v)w="reachable";else if(b)w="skipped";l[p]={id:f.id,url:m,status:w,httpStatus:y.status,reason:b?"bot-protected-or-head-disallowed":void 0}}catch(y){clearTimeout(g),l[p]={id:f.id,url:m,status:"dead",error:y.name==="AbortError"?"timeout":y.message}}}catch(h){l[p]={id:f.id,url:m,status:"dead",error:h.message}}}catch(m){l[p]={id:"?",url:"",status:"dead",error:m?.message||"unknown"}}}}let d=Math.min(i.length,n);await Promise.all(Array.from({length:d},()=>u()));for(let p of l)if(p.status==="reachable")a.push(p);else if(p.status==="dead")o.push(p);else s.push(p);return{reachable:a,dead:o,skipped:s,ok:o.length===0}}async function Gr(e,t=null){process.stderr.write(`PROGRESS:research:check-urls
|
|
418
|
+
`);try{let r=new Set(t?.cited||[]),n=r.size?(e||[]).filter((a)=>r.has(a?.id)):e,i=await qo(n,{timeoutMs:6000,concurrency:4});if(!i.ok)process.stderr.write(`[greedysearch] ${i.dead.length} dead citation URL(s) detected
|
|
419
419
|
`);return i}catch(r){return process.stderr.write(`[greedysearch] URL reachability check failed: ${r.message}
|
|
420
|
-
`),null}}function Fr({sources:
|
|
421
|
-
`)}function
|
|
422
|
-
`):
|
|
423
|
-
`),"utf8")}async function Jr({query:
|
|
420
|
+
`),null}}function Fr({sources:e=[],fetchedSources:t=[],synthesis:r={},citationAudit:n=null,gaps:i=[],questions:a=[],rounds:o=[],qualityScore:s=0,qualityThreshold:l=8.5,maxSources:c=8,requireCitations:u=!0,requireQuestions:d=!0}={}){let p=t.filter((S)=>S?.fetch?.ok||(S?.contentChars||0)>100||String(S?.content||"").length>100),f=e.filter((S)=>["official-docs","repo","maintainer-blog","academic"].includes(String(S?.sourceType||""))),m=Array.isArray(r?.claims)?r.claims:[],h=n?n.cited?.length||0:0,g=tr(a),y=(a||[]).filter((S)=>!S.createdRound||S.reason==="Original research question"),v=tr(y),b=(o||[]).length,w=Math.min(4,Math.max(2,Number(c)||8)),E=b<=1?Math.min(2,w):w,A={roundsRun:o.length>=1,fetchedSources:p.length>=E,primarySources:f.length>=1,qualityScore:s>=Math.min(l,8)||u&&m.length>0&&h>0,claimsExtracted:!u||m.length>0,citationsPresent:!u||h>0,citationsValid:!u||n?.ok===!0,unfetchedCitations:!u||(n?.unfetched||[]).length===0,requiredQuestionsClosed:!d||v.open===0};return{floorMet:Object.values(A).every(Boolean),checks:A,metrics:{fetchedOk:p.length,primarySources:f.length,claims:m.length,cited:h,gaps:i.length,openQuestions:g.open,closedQuestions:g.closed,totalQuestions:g.total,openRequiredQuestions:v.open,closedRequiredQuestions:v.closed,totalRequiredQuestions:v.total,qualityScore:s,minFetched:E}}}function Hr(e){return[{id:"Q1",question:z(Ir(e),320),status:"open",reason:"Original research question",evidence:[],sourceIds:[]}]}function Jo(e,t){let r=Ir(t).toLowerCase();return(e||[]).find((n)=>n.question?.toLowerCase()===r||jr(n.question||"",r)>=0.82)}function Yo(e,t,{evidence:r="",sourceIds:n=[],round:i=null}={}){let a=e.find((o)=>o.id===t)||Jo(e,t);if(!a)return null;if(a.status="closed",a.closedRound=a.closedRound||i,r)a.evidence=se([...a.evidence||[],r],4);if(Array.isArray(n))a.sourceIds=se([...a.sourceIds||[],...n],8);return a}function tr(e){let t=e.length,r=e.filter((n)=>n.status==="closed").length;return{total:t,closed:r,open:Math.max(0,t-r)}}function qr(e,t,r){if(!t?.answer||r?.ok!==!0)return e;let n=Array.isArray(t.claims)?t.claims:[],i=Array.isArray(r.cited)?r.cited:[];if(n.length===0||i.length===0)return e;for(let a of e){if(a.status==="closed")continue;let o=null,s=0;for(let l of n){let c=jr(a.question||"",l.claim||"");if(c>s)s=c,o=l}if(a.id==="Q1"||s>=0.18)Yo(e,a.id,{evidence:o?.claim||"Answered in final cited synthesis",sourceIds:Array.isArray(o?.sourceIds)?o.sourceIds:i.slice(0,4)})}return e}function Wo(e){if(!e.length)return"No tracked questions.";return e.map((t)=>{let r=t.sourceIds?.length?` (${t.sourceIds.join(", ")})`:"";return`- [${t.status==="closed"?"x":" "}] ${t.id}: ${t.question}${r}`}).join(`
|
|
421
|
+
`)}function Ve(e,t="None recorded."){let r=se(e);return r.length?r.map((n)=>`- ${n}`).join(`
|
|
422
|
+
`):t}function Bo(e,{query:t,rounds:r,sources:n,fetchedSources:i,citationAudit:a,citationUrls:o,floor:s,manifest:l}){let c=(i||[]).filter((h)=>h?.contentChars>100||h?.fetch?.ok),u=(n||[]).filter((h)=>["official-docs","repo","maintainer-blog","academic"].includes(String(h?.sourceType||""))),d=new Set(a?.cited||[]),p=(n||[]).filter((h)=>d.has(h?.id)),f=[`# Provenance: ${t}`,"",`- **Date:** ${l?.startedAt||new Date().toISOString()}`,`- **Duration:** ${l?.durationMs?`${(l.durationMs/1000).toFixed(1)}s`:"unknown"}`,`- **Mode:** ${l?.terminationReason==="simple_single_pass"?"simple (single-pass)":"iterative"}`,`- **Rounds:** ${l?.rounds||r?.length||1}`,"","## Sources","",`- **Consulted:** ${n?.length||0}`,`- **Fetched successfully:** ${c.length}`,`- **Primary sources:** ${u.length}`,`- **Cited in report:** ${p.length}`,""];if(p.length>0){f.push("### Cited sources","");for(let h of p){let g=h.canonicalUrl||h.finalUrl||h.url||"",y=h.fetch?.ok?"✓":"✗";f.push(`- **${h.id}:** [${h.title||g}](${g}) (${h.sourceType||"unknown"}, fetched: ${y})`)}f.push("")}if(o&&(o.reachable.length>0||o.dead.length>0)){if(f.push("## URL reachability",""),o.dead.length>0){f.push(""),f.push("**Dead links:**");for(let h of o.dead)f.push(`- ${h.id}: ${h.url} (${h.httpStatus||h.error||"unknown"})`)}if(o.reachable.length>0)f.push(""),f.push(`**Reachable:** ${o.reachable.length}/${o.reachable.length+o.dead.length}`);f.push("")}let m=!a?"NOT CHECKED":a.ok&&(o?.ok??!0)?"PASS":a.ok===!1?"FAIL (missing citations)":"FAIL (dead links)";if(f.push("## Verification","",`- **Citations:** ${a?.ok?"PASS":`FAIL — missing: ${(a?.missing||[]).join(", ")}`}`,`- **URL reachability:** ${o?o.ok?"PASS":`FAIL — ${o.dead.length} dead`:"SKIPPED"}`,`- **Floor:** ${s?.floorMet?"PASS":"PARTIAL"}`,`- **Overall:** ${m}`,""),s?.checks){f.push("## Floor checks","");for(let[h,g]of Object.entries(s.checks))f.push(`- [${g?"x":" "}] ${h}`);f.push("")}C(_(e,"provenance.md"),f.join(`
|
|
423
|
+
`),"utf8")}async function Jr({query:e,rounds:t,sources:r,fetchedSources:n,evidenceItems:i=[],synthesis:a,citationAudit:o,floor:s,manifest:l,allGaps:c=[],questions:u=[],citationUrls:d=null,outDir:p=null}){let f=new Date().toISOString().replaceAll(/[:.]/g,"-").slice(0,19),m=p||_(Io,`${f}_${jo(e)}`),h=_(m,"reports"),g=_(m,"sources"),y=_(m,"data");Qe(h,{recursive:!0}),Qe(g,{recursive:!0}),Qe(y,{recursive:!0});let v=await To(n,g),b=se([...c,...t.flatMap((w)=>w.gaps||[])]);C(_(m,"STATUS.md"),[s.floorMet?"STATUS: DONE":"STATUS: PARTIAL","",`Query: ${e}`,`Stop reason: ${l.terminationReason||"max_rounds"}`,"","## Deterministic floor checks",...Object.entries(s.checks).map(([w,E])=>`- [${E?"x":" "}] ${w}`),"","## Questions",Wo(u),"","## Open gaps",Ve(b),""].join(`
|
|
424
424
|
`),"utf8"),C(_(m,"OUTLINE.md"),["# Research bundle outline","","- `reports/SUMMARY.md` — final cited report","- `reports/CLAIMS.md` — extracted claims with support/source IDs","- `reports/EVIDENCE.md` — goal-based source evidence","- `reports/GAPS.md` — remaining caveats and uncertainties","- `provenance.md` — human-readable run metadata and verification","- `sources/` — fetched source markdown files","- `data/manifest.json` — machine-readable run metadata","- `data/rounds.json` — per-round actions/learnings/gaps","- `data/sources.json` — ranked source registry","- `data/questions.json` — open/closed question ledger",""].join(`
|
|
425
|
-
`),"utf8"),C(_(
|
|
426
|
-
`),"utf8"),C(_(
|
|
425
|
+
`),"utf8"),C(_(h,"SUMMARY.md"),String(a.answer||""),"utf8"),C(_(h,"CLAIMS.md"),["# Key claims","",...Array.isArray(a.claims)&&a.claims.length?a.claims.map((w)=>{let E=Array.isArray(w.sourceIds)?w.sourceIds.join(", "):"";return`- ${w.claim||""} (${w.support||"support unknown"}${E?`; ${E}`:""})`}):["No structured claims were extracted."],""].join(`
|
|
426
|
+
`),"utf8"),C(_(h,"EVIDENCE.md"),["# Extracted evidence","",...i.length?i.map((w)=>[`## ${w.sourceId||w.url||"Source"}`,w.url?`<${w.url}>`:"",w.rational?`**Rational:** ${w.rational}`:"",w.evidence?`**Evidence:** ${w.evidence}`:"",w.summary?`**Summary:** ${w.summary}`:"",""].filter(Boolean).join(`
|
|
427
427
|
`)):["No goal-based evidence was extracted."],""].join(`
|
|
428
|
-
`),"utf8"),C(_(
|
|
429
|
-
`),"utf8"),C(_(y,"manifest.json"),JSON.stringify({...l,floor:s,citationAudit:o},null,2),"utf8"),C(_(y,"rounds.json"),JSON.stringify(
|
|
430
|
-
`),"utf8");try{
|
|
431
|
-
`)}return{dir:m,statusPath:_(m,"STATUS.md"),summaryPath:_(
|
|
432
|
-
`)
|
|
428
|
+
`),"utf8"),C(_(h,"GAPS.md"),["# Gaps and caveats","","## Caveats",Ve(a.caveats||[]),"","## Research gaps",Ve(b),""].join(`
|
|
429
|
+
`),"utf8"),C(_(y,"manifest.json"),JSON.stringify({...l,floor:s,citationAudit:o},null,2),"utf8"),C(_(y,"rounds.json"),JSON.stringify(t,null,2),"utf8"),C(_(y,"sources.json"),JSON.stringify(r,null,2),"utf8"),C(_(y,"questions.json"),JSON.stringify(u,null,2),"utf8"),C(_(y,"evidence.json"),JSON.stringify(i,null,2),"utf8"),C(_(g,"index.md"),["# Source index","",...v.map((w)=>{let E=w.title||w.url,A=w.finalUrl||w.url,S=w.contentPath?` — ${w.contentPath}`:"";return`- ${w.id||"?"}: [${E}](${A})${S}`}),""].join(`
|
|
430
|
+
`),"utf8");try{Bo(m,{query:e,rounds:t,sources:r,fetchedSources:n,citationAudit:o,citationUrls:d,floor:s,manifest:l})}catch(w){process.stderr.write(`[greedysearch] Provenance sidecar write failed (non-critical): ${w.message}
|
|
431
|
+
`)}return{dir:m,statusPath:_(m,"STATUS.md"),summaryPath:_(h,"SUMMARY.md"),manifestPath:_(y,"manifest.json"),provenancePath:_(m,"provenance.md"),sourceCount:v.length,sourceFiles:v}}function Qo(e){let t="",r=!1,n=!1;for(let i of String(e)){if(n){t+=i,n=!1;continue}if(i==="\\"){t+=i,n=!0;continue}if(i==='"'){r=!r,t+=i;continue}if(r&&i===`
|
|
432
|
+
`)t+="\\n";else if(r&&i==="\r")t+="\\r";else if(r&&i==="\t")t+="\\t";else t+=i}return t}function xe(e){if(!e)return null;let t=String(e).trim(),r=t.indexOf("BEGIN_JSON"),n=t.indexOf("END_JSON");if(r!==-1&&n!==-1&&r<n)t=t.slice(r+10,n).trim();else{let s=t.indexOf("{");if(s>0)t=t.slice(s)}let i=[t,t.replace(/^```json\s*/i,"").replace(/^```\s*/i,"").replace(/```$/i,"").trim()],a=t.indexOf("{"),o=t.lastIndexOf("}");if(a!==-1&&o!==-1&&a<o)i.push(t.slice(a,o+1));for(let s of[...i]){let l=Qo(s);if(l!==s)i.push(l)}for(let s of i)try{return JSON.parse(s)}catch{}return null}import{mkdirSync as Ko,writeFileSync as Vo}from"node:fs";import{join as Yr}from"node:path";var zo=Yr(process.cwd(),".dm","greedysearch-sources");function wt(e,t=zo){return Ko(t,{recursive:!0}),e.map((r)=>{if(!r.content||r.content.length<10)return r;let n=String(r.id||"unknown").replace(/[^a-zA-Z0-9_-]/g,""),i=(r.canonicalUrl||r.url||"").replace(/^https?:\/\//,"").replace(/[^a-zA-Z0-9]/g,"-").slice(0,40),a=`${n}-${i}.md`,o=Yr(t,a),s=`---
|
|
433
433
|
url: ${r.finalUrl||r.url}
|
|
434
434
|
title: ${r.title||""}
|
|
435
435
|
source: ${r.source||"unknown"}
|
|
@@ -437,49 +437,49 @@ status: ${r.status||""}
|
|
|
437
437
|
chars: ${r.contentChars||r.content.length}
|
|
438
438
|
---
|
|
439
439
|
|
|
440
|
-
`;
|
|
440
|
+
`;Vo(o,s+r.content,"utf8");let{content:l,...c}=r;return{...c,contentPath:o,contentChars:r.contentChars||l.length}})}import{createRequire as Zo}from"node:module";import{createRequire as Xo}from"node:module";import{spawn as $s}from"node:child_process";import{basename as _s}from"node:path";import{dirname as Es,join as Ss}from"node:path";import{fileURLToPath as Rs}from"node:url";import{fileURLToPath as Ds}from"node:url";import{existsSync as je,mkdirSync as As,readFileSync as dn,writeFileSync as Cs}from"node:fs";import{homedir as ks}from"node:os";import{join as fn}from"node:path";import{tmpdir as Os}from"node:os";import{existsSync as Te,mkdirSync as Ns,readFileSync as pn,writeFileSync as Ps}from"node:fs";import{homedir as xs}from"node:os";import{join as hn}from"node:path";import{tmpdir as Is}from"node:os";import{existsSync as Le,mkdirSync as Zs,readFileSync as bn,writeFileSync as Xs}from"node:fs";import{homedir as el}from"node:os";import{join as vn}from"node:path";import{tmpdir as tl}from"node:os";var mu=Zo(import.meta.url),wu=Xo(import.meta.url),Ie=null;async function es(){if(Ie)return Ie;let[{Readability:e},{JSDOM:t},{default:r}]=await Promise.all([import("@mozilla/readability"),import("jsdom"),import("turndown")]),n=new r({headingStyle:"atx",bulletListMarker:"-",codeBlockStyle:"fenced"});return n.addRule("removeDataUrls",{filter:(i)=>i.tagName==="IMG"&&i.getAttribute("src")?.startsWith("data:"),replacement:()=>""}),Ie={Readability:e,JSDOM:t,turndown:n},Ie}var rn="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",nn={"user-agent":rn,accept:"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8","accept-language":"en-US,en;q=0.9","accept-encoding":"gzip, deflate, br","cache-control":"no-cache",pragma:"no-cache","sec-ch-ua":'"Chromium";v="122", "Not(A:Brand";v="24", "Google Chrome";v="122"',"sec-ch-ua-mobile":"?0","sec-ch-ua-platform":'"Windows"',"sec-fetch-dest":"document","sec-fetch-mode":"navigate","sec-fetch-site":"none","sec-fetch-user":"?1","upgrade-insecure-requests":"1"},an=[/^localhost$/i,/^127\.\d+\.\d+\.\d+$/,/^0\.0\.0\.0$/,/^\[::1\]$/,/^10\./,/^172\.(1[6-9]|2\d|3[01])\./,/^192\.168\./,/^169\.254\./,/^fc00:/i,/^fe80:/i,/\.local$/i,/\.internal$/i,/\.localhost$/i],ts=/^::ffff:(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/i,rs=/^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/i,ns=/^(?:0x[0-9a-f]+|0[0-7]*|[1-9][0-9]*)$/i;function is(e){if(!e||e.includes(":"))return null;let t=e.split(".");if(t.length<1||t.length>4)return null;if(!t.every((i)=>ns.test(i)))return null;let r=t.map((i)=>{if(/^0x/i.test(i))return parseInt(i,16);if(/^0[0-7]+$/.test(i))return parseInt(i,8);return parseInt(i,10)});if(r.some((i)=>!Number.isFinite(i)||i<0))return null;let n;if(r.length===1){if(r[0]>4294967295)return null;n=[r[0]>>>24&255,r[0]>>>16&255,r[0]>>>8&255,r[0]&255]}else if(r.length===2){if(r[0]>255||r[1]>16777215)return null;n=[r[0],r[1]>>>16&255,r[1]>>>8&255,r[1]&255]}else if(r.length===3){if(r[0]>255||r[1]>255||r[2]>65535)return null;n=[r[0],r[1],r[2]>>>8&255,r[2]&255]}else{if(r.some((i)=>i>255))return null;n=r}return n.join(".")}function as(e){let t=e.match(ts);if(t)return`${t[1]}.${t[2]}.${t[3]}.${t[4]}`;let r=e.match(rs);if(r){let n=parseInt(r[1],16),i=parseInt(r[2],16);if(n>65535||i>65535)return null;return[n>>>8&255,n&255,i>>>8&255,i&255].join(".")}return null}function Wr(e){return an.some((t)=>t.test(e))}function os(e={}){return{...nn,...e}}function _t(e){try{if(typeof e!=="string"||!e.trim())return{blocked:!0,reason:"URL must be a non-empty string"};let t=new URL(e);if(t.protocol!=="http:"&&t.protocol!=="https:")return{blocked:!0,reason:`Protocol not allowed: ${t.protocol}`};let r=t.hostname.toLowerCase();for(let i of an)if(i.test(r))return{blocked:!0,reason:`Private/internal address: ${r}`};if(r.startsWith("[")&&r.endsWith("]")){let i=r.slice(1,-1),a=as(i);if(a&&Wr(a))return{blocked:!0,reason:`Private/internal address: ${r} (maps to ${a})`}}let n=is(r);if(n&&Wr(n))return{blocked:!0,reason:`Private/internal address: ${r} (normalizes to ${n})`};return{blocked:!1}}catch(t){return{blocked:!0,reason:`Invalid URL: ${t.message}`}}}function ss(e){try{let t=new URL(e);if(!(t.hostname==="github.com"||t.hostname.endsWith(".github.com")))return e;let r=t.pathname.split("/").filter(Boolean);if(r.length<5)return e;let[n,i,a,o,...s]=r;if(a!=="blob")return e;let l=s.join("/");return`https://raw.githubusercontent.com/${n}/${i}/${o}/${l}`}catch{return e}}async function ls(e,t={}){let r=_t(e);if(r.blocked)return{ok:!1,url:e,finalUrl:e,status:403,error:`Blocked: ${r.reason}`,needsBrowser:!1};let n=e;if(e=ss(e),e!==n)console.error(`[fetcher] Rewrote GitHub URL: ${n.slice(0,60)}... → raw.githubusercontent.com`);let{timeoutMs:i=15000,userAgent:a,signal:o}=t,s=new AbortController,l=setTimeout(()=>s.abort(),i);if(o)o.addEventListener("abort",()=>s.abort(),{once:!0});try{let c=await fetch(e,{method:"GET",headers:{...nn,"user-agent":a||rn},redirect:"follow",signal:s.signal});clearTimeout(l);let u=c.headers.get("content-type")||"",d=c.url,p=c.headers.get("last-modified")||"",f=_t(d);if(f.blocked)return{ok:!1,url:e,finalUrl:d,status:c.status,error:`Blocked: ${f.reason}`,needsBrowser:!1};let m=!1;try{m=new URL(d).hostname.toLowerCase()==="raw.githubusercontent.com"}catch{}if(u.includes("text/plain")&&m){let b=await c.text();return{ok:!0,url:n,finalUrl:d,status:c.status,title:d.split("/").pop()||"GitHub File",byline:"",siteName:"GitHub",lang:"",publishedTime:p,lastModified:p,markdown:b,contentLength:b.length,excerpt:b.slice(0,300).replaceAll(/\n/g," "),needsBrowser:!1}}if(!u.includes("text/html")&&!u.includes("application/xhtml"))return{ok:!1,url:e,finalUrl:d,status:c.status,error:`Unsupported content type: ${u}`,needsBrowser:!1};let h=await c.text(),g=on(c.status,h,d,e);if(g.blocked)return{ok:!1,url:e,finalUrl:d,status:c.status,error:`Blocked: ${g.reason}`,needsBrowser:!0};let y=await sn(h,d),v=ln(y);if(!v.ok)return{ok:!1,url:e,finalUrl:d,status:c.status,error:`Low quality content: ${v.reason}`,needsBrowser:!0};return{ok:!0,url:e,finalUrl:d,status:c.status,title:y.title,byline:y.byline,siteName:y.siteName,lang:y.lang,publishedTime:y.publishedTime||p,lastModified:p,markdown:y.markdown,excerpt:y.excerpt,contentLength:y.markdown.length,needsBrowser:!1}}catch(c){clearTimeout(l);let u=ps(c);return{ok:!1,url:e,finalUrl:e,status:0,error:c.message,needsBrowser:u}}}function on(e,t,r,n){let i=t.match(/<title[^>]*>([^<]*)<\/title>/i)?.[1]?.toLowerCase()||"",a=t.slice(0,30000).toLowerCase(),o=`${i} ${a}`;if(e===403||e===429||e===503)return{blocked:!0,reason:`HTTP ${e}`};let s=[{pattern:/class=["'][^"']*captcha["']|<div[^>]*id=["']captcha/i,reason:"captcha"},{pattern:/g-recaptcha|data-sitekey|i['"]m not a robot/i,reason:"captcha"},{pattern:/checking your browser.{0,100}please wait|cf-browser-verification/i,reason:"cloudflare challenge"},{pattern:/just a moment.{0,50}security check|ddos protection by cloudflare/i,reason:"cloudflare challenge"},{pattern:/unusual traffic.{0,50}from your computer network/i,reason:"unusual traffic"},{pattern:/bot detected|automated.{0,20}request/i,reason:"bot detection"},{pattern:/enable\s+javascript\s+to\s+view|javascript\s+is\s+required.{0,50}enabled/i,reason:"requires javascript"},{pattern:/access denied|accessdenied/i,reason:"access denied"},{pattern:/protected by anubis|anubis uses a proof-of-work/i,reason:"anubis challenge"}];for(let c of s)if(c.pattern.test(o))return{blocked:!0,reason:c.reason};let l=fs(n,r,t);if(l)return{blocked:!0,reason:l};return{blocked:!1}}var cs=["accounts.google.com","login.microsoftonline.com","login.live.com","auth0.com","okta.com","auth.mozilla.auth0.com","id.atlassian.com"],us=["login.","signin.","auth.","sso.","accounts.","idp."],ds=["sign in to continue","log in to continue","authentication required","create an account to continue","subscribe to continue reading","members only"];function fs(e,t,r){try{let n=new URL(e),i=new URL(t);if(n.hostname.toLowerCase()===i.hostname.toLowerCase())return;let a=i.hostname.toLowerCase();if(cs.some((s)=>a===s||a.endsWith(`.${s}`)))return`redirected to login (${i.hostname})`;if(us.some((s)=>a.startsWith(s)))return`redirected to login (${i.hostname})`;let o=r.slice(0,20000).toLowerCase();if(ds.some((s)=>o.includes(s)))return`redirected to login page (${i.hostname})`}catch{}return}function ps(e){let t=e.message.toLowerCase();return t.includes("fetch failed")||t.includes("unable to verify")||t.includes("certificate")||t.includes("timeout")}function Br(e){let t=['meta[property="article:published_time"]','meta[name="article:published_time"]','meta[property="og:published_time"]','meta[name="publication_date"]','meta[name="date"]','meta[itemprop="datePublished"]','time[itemprop="datePublished"]','meta[name="DC.date"]'];for(let r of t){let n=e.querySelector(r),i=n?.getAttribute("content")||n?.getAttribute("datetime")||"";if(i)return i}return""}async function sn(e,t){let{Readability:r,JSDOM:n,turndown:i}=await es(),a=new n(e,{url:t});try{let o=a.window.document,s=new r(o).parse();if(s&&s.content){let c=i.turndown(s.content).replaceAll(/\n{3,}/g,`
|
|
441
441
|
|
|
442
|
-
`).trim(),u=s.publishedTime||Br(o)||"";return{title:s.title||o.title||
|
|
442
|
+
`).trim(),u=s.publishedTime||Br(o)||"";return{title:s.title||o.title||t,byline:s.byline||"",siteName:s.siteName||"",lang:s.lang||"",publishedTime:u,markdown:c,excerpt:c.slice(0,300).replaceAll(/\n/g," ")}}let l=o.body;if(l){let c=l.cloneNode(!0);c.querySelectorAll("script, style, nav, footer, header, aside").forEach((d)=>d.remove());let u=(c.textContent||"").replaceAll(/\s+/g," ").trim();return{title:o.title||t,byline:"",siteName:"",lang:"",publishedTime:Br(o),markdown:u,excerpt:u.slice(0,300)}}return{title:t,byline:"",siteName:"",lang:"",publishedTime:"",markdown:"",excerpt:""}}finally{a.window.close()}}function ln(e){let t=e.markdown.trim().toLowerCase(),r=(e.title||"").toLowerCase();if(e.markdown.trim().length<100)return{ok:!1,reason:"content too short (< 100 chars)"};let n=t.toLowerCase(),i=[{check:()=>n.includes("loading")&&n.includes("please wait"),desc:"loading page"},{check:()=>n.includes("please ensure javascript is enabled"),desc:"requires javascript"},{check:()=>n.includes("enable javascript to view"),desc:"requires javascript"},{check:()=>n.includes("just a moment"),desc:"cloudflare challenge detected in content"},{check:()=>n.includes("verify you are human"),desc:"human verification"},{check:()=>n.includes("captcha required"),desc:"captcha in extracted content"},{check:()=>n.includes("access denied"),desc:"access denied in content"},{check:()=>/^\s{0,10}sign\s{1,5}in\s{0,10}$|^\s{0,10}log\s{1,5}in\s{0,10}$/im.test(t),desc:"login form only"}];for(let{check:a,desc:o}of i)if(a())return{ok:!1,reason:o};if(r.includes("just a moment")||r.includes("checking your browser"))return{ok:!1,reason:"cloudflare challenge page detected in title"};return{ok:!0}}var cn={"user-agent":"GreedySearch/1.0",accept:"application/vnd.github+json","x-github-api-version":"2022-11-28"};function Et(e){try{let t=new URL(e);if(!(t.hostname==="github.com"||t.hostname.endsWith(".github.com")))return null;let r=t.pathname.split("/").filter(Boolean);if(r.length<2)return null;let[n,i]=r;if(r.length===2)return{owner:n,repo:i,type:"root"};if(r.length>=4&&(r[2]==="blob"||r[2]==="tree")){let a=r[2],o=r[3],s=r.slice(4).join("/");return{owner:n,repo:i,type:a,ref:o,path:s}}return null}catch{return null}}async function T(e,t=1e4){let r=new AbortController,n=setTimeout(()=>r.abort(),t);try{let i=await fetch(`https://api.github.com${e}`,{headers:cn,signal:r.signal});if(clearTimeout(n),!i.ok)throw Error(`GitHub API ${i.status}: ${e}`);return await i.json()}catch(i){throw clearTimeout(n),i}}async function hs(e,t){try{let r=await T(`/repos/${e}/${t}/readme`);if(r.content&&r.encoding==="base64")return Buffer.from(r.content,"base64").toString("utf8");return""}catch{return""}}async function Qr(e,t,r="HEAD",n="",i){try{let a;if(r==="HEAD")if(i)a=await T(`/repos/${e}/${t}/git/ref/heads/${i}`).catch(()=>null);else a=await Promise.any([T(`/repos/${e}/${t}/git/ref/heads/main`),T(`/repos/${e}/${t}/git/ref/heads/master`)]).catch(()=>null);else a=await T(`/repos/${e}/${t}/git/ref/heads/${r}`).catch(()=>T(`/repos/${e}/${t}/git/ref/heads/master`).catch(()=>null));if(!a?.object?.sha)return[];let o=(await T(`/repos/${e}/${t}/git/commits/${a.object.sha}`)).tree.sha,s=(await T(`/repos/${e}/${t}/git/trees/${o}`)).tree||[];if(n)s=s.filter((l)=>l.path.startsWith(n));return s.slice(0,50).map((l)=>({path:l.path,type:l.type==="tree"?"dir":"file",size:l.size}))}catch{return[]}}async function gs(e,t,r,n,i=1e4,a){let o=async(l)=>{let c=new AbortController,u=setTimeout(()=>c.abort(),i);try{let d=await fetch(l,{headers:{"user-agent":cn["user-agent"]},signal:c.signal});if(clearTimeout(u),d.ok)return await d.text();throw Error("not ok")}catch{throw clearTimeout(u),Error("failed")}};if(!r||r==="HEAD"){if(a)try{return await o(`https://raw.githubusercontent.com/${e}/${t}/${a}/${n}`)}catch{return null}try{return await Promise.any([o(`https://raw.githubusercontent.com/${e}/${t}/main/${n}`),o(`https://raw.githubusercontent.com/${e}/${t}/master/${n}`)])}catch{return null}}let s=[`https://raw.githubusercontent.com/${e}/${t}/${r}/${n}`,`https://raw.githubusercontent.com/${e}/${t}/master/${n}`];for(let l of s)try{return await o(l)}catch{}return null}async function ms(e){let t=Et(e);if(!t)return{ok:!1,error:"Not a valid GitHub URL"};let{owner:r,repo:n,type:i,ref:a,path:o}=t;try{if(i==="root"||i==="tree"&&!o){let s=await T(`/repos/${r}/${n}`),[l,c]=await Promise.allSettled([hs(r,n),Qr(r,n,a||"HEAD","",s.default_branch)]),u=l.status==="fulfilled"?l.value:"",d=c.status==="fulfilled"?c.value:[],p=s?.description?`
|
|
443
443
|
|
|
444
|
-
> ${s.description}`:"",f=s?.stargazers_count==null?"":` ⭐ ${s.stargazers_count}`,m=s?.language?` · ${s.language}`:"",
|
|
444
|
+
> ${s.description}`:"",f=s?.stargazers_count==null?"":` ⭐ ${s.stargazers_count}`,m=s?.language?` · ${s.language}`:"",h=`# ${r}/${n}${f}${m}${p}
|
|
445
445
|
|
|
446
|
-
`;if(u)
|
|
446
|
+
`;if(u)h+=u.slice(0,6000);else h+=`[No README found]
|
|
447
447
|
|
|
448
448
|
Files:
|
|
449
|
-
${d.map((
|
|
450
|
-
`)}`;return{ok:!0,title:`${r}/${n}`,content:
|
|
449
|
+
${d.map((g)=>` ${g.type==="dir"?"\uD83D\uDCC1":"\uD83D\uDCC4"} ${g.path}`).join(`
|
|
450
|
+
`)}`;return{ok:!0,title:`${r}/${n}`,content:h,tree:d.slice(0,30)}}if(i==="blob"&&o){let s;if(!a||a==="HEAD")try{s=(await T(`/repos/${r}/${n}`)).default_branch}catch{s=void 0}let l=await gs(r,n,a,o,1e4,s);if(l===null)return{ok:!1,error:`File not found: ${o}`};return{ok:!0,title:`${r}/${n}: ${o}`,content:l}}if(i==="tree"&&o){let s;if(!a||a==="HEAD")try{s=(await T(`/repos/${r}/${n}`)).default_branch}catch{s=void 0}let l=await Qr(r,n,a||"HEAD",o,s),c=l.map((u)=>` ${u.type==="dir"?"\uD83D\uDCC1":"\uD83D\uDCC4"} ${u.path}`).join(`
|
|
451
451
|
`);return{ok:!0,title:`${r}/${n}/${o}`,content:`[Directory: ${o}]
|
|
452
452
|
|
|
453
453
|
Files:
|
|
454
|
-
${c}`,tree:l}}return{ok:!1,error:"Unsupported GitHub URL type"}}catch(s){return{ok:!1,error:s.message}}}var
|
|
454
|
+
${c}`,tree:l}}return{ok:!1,error:"Unsupported GitHub URL type"}}catch(s){return{ok:!1,error:s.message}}}var ys={"user-agent":"GreedySearch/1.0 (Research Bot)",accept:"application/json"};function ws(e){try{let t=new URL(e),r=t.hostname.toLowerCase();if(!(r==="reddit.com"||r.endsWith(".reddit.com")))return null;let n=t.pathname;if(n.match(/^\/(u|user)\/[^/]+\/?$/i))return{type:"user",cleanUrl:Kr(e)};if(n.match(/^\/r\/[^/]+\/comments\/[^/]+/i))return{type:"post",cleanUrl:Kr(e)};return null}catch{return null}}function Kr(e){try{let t=new URL(e);return`${t.protocol}//${t.hostname}${t.pathname}`}catch{return e}}async function bs(e,t=8000){let r=Date.now();try{let n=e.replace(/\/+$/,"")+".json",i=new AbortController,a=setTimeout(()=>i.abort(),15000),o=await fetch(n,{headers:ys,signal:i.signal});if(clearTimeout(a),!o.ok)throw Error(`Reddit API ${o.status}`);let s=await o.json();if(!Array.isArray(s)||s.length<1)throw Error("Invalid Reddit API response structure");let l=s[0],c=s[1],u=l?.data?.children?.[0]?.data;if(!u)throw Error("No post data in Reddit response");let d=vs(u,c,t);return{ok:!0,url:e,finalUrl:e,status:200,contentType:"text/markdown",lastModified:"",title:u.title||"Reddit Post",byline:`u/${u.author}`,siteName:`r/${u.subreddit}`,lang:"en",publishedTime:new Date(u.created_utc*1000).toISOString(),excerpt:u.selftext?.slice(0,300).replace(/\n/g," ")||"",markdown:d,contentLength:d.length,needsBrowser:!1,duration:Date.now()-r}}catch(n){return{ok:!1,url:e,finalUrl:e,status:0,error:`Reddit fetch failed: ${n.message}`,needsBrowser:!1,duration:Date.now()-r}}}function vs(e,t,r){let n="";if(n+=`# ${e.title}
|
|
455
455
|
|
|
456
|
-
`,n+=`**Subreddit:** r/${
|
|
456
|
+
`,n+=`**Subreddit:** r/${e.subreddit} | **Author:** u/${e.author} | **Score:** ${e.score}
|
|
457
457
|
|
|
458
|
-
`,
|
|
458
|
+
`,e.selftext)n+=e.selftext,n+=`
|
|
459
459
|
|
|
460
|
-
`;else if(
|
|
460
|
+
`;else if(e.url)try{let i=new URL(e.url).hostname.toLowerCase();if(i!=="reddit.com"&&!i.endsWith(".reddit.com"))n+=`**Link:** ${e.url}
|
|
461
461
|
|
|
462
|
-
`}catch{n+=`**Link:** ${
|
|
462
|
+
`}catch{n+=`**Link:** ${e.url}
|
|
463
463
|
|
|
464
|
-
`}if(
|
|
464
|
+
`}if(t?.data?.children?.length>0){n+=`---
|
|
465
465
|
|
|
466
466
|
## Comments
|
|
467
467
|
|
|
468
|
-
`;let i=
|
|
468
|
+
`;let i=t.data.children.filter((a)=>a.kind==="t1").slice(0,10);for(let a of i)n+=un(a.data,0),n+=`
|
|
469
469
|
`}if(n.length>r)n=n.slice(0,r).trim()+`
|
|
470
470
|
|
|
471
|
-
... (truncated)`;return n}function un(t
|
|
472
|
-
`,n+=`${r}${
|
|
471
|
+
... (truncated)`;return n}function un(e,t){if(!e||e.body==="[deleted]"||e.body==="[removed]")return"";let r="> ".repeat(t),n="";if(n+=`${r}**u/${e.author}** (${e.score} pts)
|
|
472
|
+
`,n+=`${r}${e.body.replaceAll(`
|
|
473
473
|
`,`
|
|
474
474
|
`+r)}
|
|
475
|
-
`,
|
|
476
|
-
`+un(a.data,
|
|
475
|
+
`,t<3&&e.replies?.data?.children){let i=e.replies.data.children.filter((a)=>a.kind==="t1");for(let a of i.slice(0,5))n+=`
|
|
476
|
+
`+un(a.data,t+1)}return n}function ne(e,t=8000){if(!e||e.length<=t)return e;let r=`
|
|
477
477
|
|
|
478
478
|
[...content trimmed...]
|
|
479
479
|
|
|
480
|
-
`,n=
|
|
481
|
-
`)o--;if(o<=i-100)o=i;let s=
|
|
482
|
-
`)s++;if(s>=
|
|
480
|
+
`,n=t-r.length,i=Math.floor(n*0.75),a=n-i,o=i;while(o>i-100&&e[o]!==`
|
|
481
|
+
`)o--;if(o<=i-100)o=i;let s=e.length-a;while(s<e.length-a+100&&e[s]!==`
|
|
482
|
+
`)s++;if(s>=e.length-a+100)s=e.length-a;let l=e.slice(0,o).trimEnd(),c=e.slice(s).trimStart();return`${l}${r}${c}`}function js(e=process.env,t=process.execPath){let r=e.GREEDY_SEARCH_NODE||e.NODE_BINARY||e.NODE;if(r?.trim())return r.trim();let n=_s(t||"").toLowerCase();if(n==="node"||n==="node.exe")return t;return"node"}var Ts=Es(Rs(import.meta.url)),Ls=Ss(Ts,"..","bin","cdp.mjs"),Ms=new Set(["list","snap","eval","shot","html","nav","net","click","clickxy","type","loadall","evalraw","browse","stop","--tab"]);function Us(e){if(!Array.isArray(e)||e.length===0)throw Error("cdp: args must be a non-empty array");if(e[0]==="test")return e.map((t,r)=>Vr(t,r));if(!Ms.has(e[0]))throw Error(`cdp: unknown subcommand '${e[0]}'`);return e.map((t,r)=>Vr(t,r))}function Vr(e,t){if(typeof e!=="string")throw Error(`cdp: argv[${t}] must be a string (got ${typeof e})`);if(e.includes("\x00"))throw Error(`cdp: argv[${t}] contains a null byte`);return e}function gn(e,t=30000){return Gs(e,null,t)}function Gs(e,t=null,r=30000){let n=Us(e);return new Promise((i,a)=>{let o=$s(js(),[Ls,...n],{stdio:[t==null?"ignore":"pipe","pipe","pipe"]});if(t!=null)o.stdin.write(t),o.stdin.end();let s="",l="";o.stdout.on("data",(u)=>s+=u),o.stderr.on("data",(u)=>l+=u);let c=setTimeout(()=>{o.kill(),a(Error(`cdp timeout: ${e[0]}`))},r);o.on("close",(u)=>{if(clearTimeout(c),u===0)i(s.trim());else a(Error(l.trim()||`cdp exit ${u}`))})})}async function zr(e){await gn(["evalraw",e,"Page.addScriptToEvaluateOnNewDocument",JSON.stringify({source:`
|
|
483
483
|
(function() {
|
|
484
484
|
// ── Runtime.enable / CDP detection masking ──────────────
|
|
485
485
|
try { delete window.__REBROWSER_RUNTIME_ENABLE; } catch(_) {}
|
|
@@ -764,35 +764,35 @@ ${c}`,tree:l}}return{ok:!1,error:"Unsupported GitHub URL type"}}catch(s){return{
|
|
|
764
764
|
};
|
|
765
765
|
} catch(_) {}
|
|
766
766
|
})();
|
|
767
|
-
`})])}function
|
|
767
|
+
`})])}function Fs(e){return Ds(new URL(".",e)).replace(/^\/([A-Z]:)/,"$1")}function Hs(e,t=9222){let r=Number.parseInt(String(e??""),10);return Number.isInteger(r)&&r>1024&&r<65535?r:t}var qs=Os().replaceAll("\\","/"),Pu=Hs(process.env.GREEDY_SEARCH_PORT),G=(process.env.GREEDY_SEARCH_PROFILE_DIR||process.env.CDP_PROFILE_DIR||`${qs}/greedysearch-chrome-profile`).replaceAll("\\","/"),xu=`${G}/DevToolsActivePort`,Iu=process.env.GREEDY_SEARCH_PID_FILE||`${G}/browser.pid`,ju=process.env.CDP_PAGES_CACHE||`${G}/cdp-pages.json`,Tu=process.env.GREEDY_SEARCH_MODE_FILE||`${G}/browser-mode`,Lu=process.env.GREEDY_SEARCH_METADATA_FILE||`${G}/browser-metadata.json`,Mu=process.env.GREEDY_SEARCH_LAUNCH_LOCK_FILE||`${G}/browser-launch.lock`,Uu=process.env.GREEDY_SEARCH_ACTIVITY_FILE||`${G}/browser-last-activity`,Gu=(process.env.CDP_SOCKET_DIR||`${G}/cdp-sockets`).replaceAll("\\","/"),Fu=`${G}/visible-recovery.jsonl`,St=fn(ks(),".dm"),Y=fn(St,"greedyconfig"),Rt=["perplexity","google","chatgpt","gemini"],Dt="gemini";function Js(){try{if(je(Y)){let e=dn(Y,"utf8"),t=JSON.parse(e);if(Array.isArray(t.engines)&&t.engines.length>0&&t.engines.every((r)=>typeof r==="string")){let r=t.engines.filter((i)=>bt[i]),n=t.engines.filter((i)=>!bt[i]);if(n.length>0)process.stderr.write(`[greedysearch] Warning: ignoring unknown engine(s) in ${Y}: ${n.join(", ")}
|
|
768
768
|
[greedysearch] Available engines: ${Object.keys(bt).join(", ")}
|
|
769
769
|
`);if(r.length>0)return r;process.stderr.write(`[greedysearch] Warning: no valid engines in ${Y}, falling back to defaults: ${Rt.join(", ")}
|
|
770
|
-
`)}}}catch{}return Rt}function
|
|
771
|
-
`,"utf8")}catch{}}
|
|
770
|
+
`)}}}catch{}return Rt}function Ys(){try{if(!je(St))As(St,{recursive:!0});if(!je(Y))Cs(Y,JSON.stringify({engines:Rt,synthesizer:Dt},null,2)+`
|
|
771
|
+
`,"utf8")}catch{}}Ys();var Zr=["gemini","chatgpt"];function Ws(){try{if(je(Y)){let e=dn(Y,"utf8"),t=JSON.parse(e);if(typeof t.synthesizer==="string"){let r=t.synthesizer.toLowerCase();if(Zr.includes(r))return r;process.stderr.write(`[greedysearch] Warning: unknown synthesizer "${t.synthesizer}" in ${Y}
|
|
772
772
|
[greedysearch] Available synthesizers: ${Zr.join(", ")}
|
|
773
773
|
[greedysearch] Falling back to default: ${Dt}
|
|
774
|
-
`)}}}catch{}return Dt}var bt={perplexity:"perplexity.mjs",p:"perplexity.mjs",bing:"bing-copilot.mjs",b:"bing-copilot.mjs",google:"google-ai.mjs",g:"google-ai.mjs",gemini:"gemini.mjs",gem:"gemini.mjs",chatgpt:"chatgpt.mjs",gpt:"chatgpt.mjs","semantic-scholar":"semantic-scholar.mjs",semanticscholar:"semantic-scholar.mjs",s2:"semantic-scholar.mjs",logically:"logically.mjs",log:"logically.mjs"},
|
|
774
|
+
`)}}}catch{}return Dt}var bt={perplexity:"perplexity.mjs",p:"perplexity.mjs",bing:"bing-copilot.mjs",b:"bing-copilot.mjs",google:"google-ai.mjs",g:"google-ai.mjs",gemini:"gemini.mjs",gem:"gemini.mjs",chatgpt:"chatgpt.mjs",gpt:"chatgpt.mjs","semantic-scholar":"semantic-scholar.mjs",semanticscholar:"semantic-scholar.mjs",s2:"semantic-scholar.mjs",logically:"logically.mjs",log:"logically.mjs"},Hu=Js(),qu=Ws(),Ju=Math.max(1,Number.parseInt(process.env.GREEDY_FETCH_CONCURRENCY||"5",10)||5);process.env.CDP_PROFILE_DIR=G;function Bs(e,t=9222){let r=Number.parseInt(String(e??""),10);return Number.isInteger(r)&&r>1024&&r<65535?r:t}var Qs=Is().replaceAll("\\","/"),Yu=Bs(process.env.GREEDY_SEARCH_PORT),F=(process.env.GREEDY_SEARCH_PROFILE_DIR||process.env.CDP_PROFILE_DIR||`${Qs}/greedysearch-chrome-profile`).replaceAll("\\","/"),Wu=`${F}/DevToolsActivePort`,Bu=process.env.GREEDY_SEARCH_PID_FILE||`${F}/browser.pid`,Qu=process.env.CDP_PAGES_CACHE||`${F}/cdp-pages.json`,Ku=process.env.GREEDY_SEARCH_MODE_FILE||`${F}/browser-mode`,Vu=process.env.GREEDY_SEARCH_METADATA_FILE||`${F}/browser-metadata.json`,zu=process.env.GREEDY_SEARCH_LAUNCH_LOCK_FILE||`${F}/browser-launch.lock`,Zu=process.env.GREEDY_SEARCH_ACTIVITY_FILE||`${F}/browser-last-activity`,Xu=(process.env.CDP_SOCKET_DIR||`${F}/cdp-sockets`).replaceAll("\\","/"),ed=`${F}/visible-recovery.jsonl`,At=hn(xs(),".dm"),W=hn(At,"greedyconfig"),Ct=["perplexity","google","chatgpt","gemini"],kt="gemini";function Ks(){try{if(Te(W)){let e=pn(W,"utf8"),t=JSON.parse(e);if(Array.isArray(t.engines)&&t.engines.length>0&&t.engines.every((r)=>typeof r==="string")){let r=t.engines.filter((i)=>vt[i]),n=t.engines.filter((i)=>!vt[i]);if(n.length>0)process.stderr.write(`[greedysearch] Warning: ignoring unknown engine(s) in ${W}: ${n.join(", ")}
|
|
775
775
|
[greedysearch] Available engines: ${Object.keys(vt).join(", ")}
|
|
776
776
|
`);if(r.length>0)return r;process.stderr.write(`[greedysearch] Warning: no valid engines in ${W}, falling back to defaults: ${Ct.join(", ")}
|
|
777
|
-
`)}}}catch{}return Ct}function
|
|
778
|
-
`,"utf8")}catch{}}
|
|
777
|
+
`)}}}catch{}return Ct}function Vs(){try{if(!Te(At))Ns(At,{recursive:!0});if(!Te(W))Ps(W,JSON.stringify({engines:Ct,synthesizer:kt},null,2)+`
|
|
778
|
+
`,"utf8")}catch{}}Vs();var Xr=["gemini","chatgpt"];function zs(){try{if(Te(W)){let e=pn(W,"utf8"),t=JSON.parse(e);if(typeof t.synthesizer==="string"){let r=t.synthesizer.toLowerCase();if(Xr.includes(r))return r;process.stderr.write(`[greedysearch] Warning: unknown synthesizer "${t.synthesizer}" in ${W}
|
|
779
779
|
[greedysearch] Available synthesizers: ${Xr.join(", ")}
|
|
780
780
|
[greedysearch] Falling back to default: ${kt}
|
|
781
|
-
`)}}}catch{}return kt}var vt={perplexity:"perplexity.mjs",p:"perplexity.mjs",bing:"bing-copilot.mjs",b:"bing-copilot.mjs",google:"google-ai.mjs",g:"google-ai.mjs",gemini:"gemini.mjs",gem:"gemini.mjs",chatgpt:"chatgpt.mjs",gpt:"chatgpt.mjs","semantic-scholar":"semantic-scholar.mjs",semanticscholar:"semantic-scholar.mjs",s2:"semantic-scholar.mjs",logically:"logically.mjs",log:"logically.mjs"},
|
|
782
|
-
`)[0];if(!
|
|
781
|
+
`)}}}catch{}return kt}var vt={perplexity:"perplexity.mjs",p:"perplexity.mjs",bing:"bing-copilot.mjs",b:"bing-copilot.mjs",google:"google-ai.mjs",g:"google-ai.mjs",gemini:"gemini.mjs",gem:"gemini.mjs",chatgpt:"chatgpt.mjs",gpt:"chatgpt.mjs","semantic-scholar":"semantic-scholar.mjs",semanticscholar:"semantic-scholar.mjs",s2:"semantic-scholar.mjs",logically:"logically.mjs",log:"logically.mjs"},td=Ks(),rd=zs(),nd=Math.max(1,Number.parseInt(process.env.GREEDY_FETCH_CONCURRENCY||"5",10)||5);process.env.CDP_PROFILE_DIR=F;var id=Number.parseInt(process.env.GREEDY_SEARCH_IDLE_TIMEOUT_MINUTES||"5",10)||5,ad=Fs(import.meta.url),od=Number.parseInt(process.env.GREEDY_SEARCH_IDLE_TIMEOUT_MINUTES||"5",10)||5,sd=Number.parseInt(process.env.GREEDY_SEARCH_VISIBLE_IDLE_TIMEOUT_MINUTES||"60",10)||60,D=gn;async function mn(){let e=(await D(["list"])).split(`
|
|
782
|
+
`)[0];if(!e)throw Error("No Chrome tabs found");return e.slice(0,8)}async function yn(e="about:blank"){let t=await mn(),r=new URL(e).hostname;if(r==="copilot.microsoft.com"||r==="www.perplexity.ai"||r==="perplexity.ai"||r.endsWith(".perplexity.ai")){let a=await D(["evalraw",t,"Target.createTarget",JSON.stringify({url:"about:blank"})]),{targetId:o}=JSON.parse(a),s=o.slice(0,8);if(await D(["list"]).catch(()=>null),r==="copilot.microsoft.com")await zr(s);else zr(s).catch(()=>{});return await D(["list"]).catch(()=>null),o}let n=await D(["evalraw",t,"Target.createTarget",JSON.stringify({url:e})]),{targetId:i}=JSON.parse(n);return await D(["list"]).catch(()=>null),i}async function wn(e){try{let t=await mn();await D(["evalraw",t,"Target.closeTarget",JSON.stringify({targetId:e})])}catch{}}function rl(e,t=9222){let r=Number.parseInt(String(e??""),10);return Number.isInteger(r)&&r>1024&&r<65535?r:t}var nl=tl().replaceAll("\\","/"),fd=rl(process.env.GREEDY_SEARCH_PORT),H=(process.env.GREEDY_SEARCH_PROFILE_DIR||process.env.CDP_PROFILE_DIR||`${nl}/greedysearch-chrome-profile`).replaceAll("\\","/"),pd=`${H}/DevToolsActivePort`,hd=process.env.GREEDY_SEARCH_PID_FILE||`${H}/browser.pid`,gd=process.env.CDP_PAGES_CACHE||`${H}/cdp-pages.json`,md=process.env.GREEDY_SEARCH_MODE_FILE||`${H}/browser-mode`,yd=process.env.GREEDY_SEARCH_METADATA_FILE||`${H}/browser-metadata.json`,wd=process.env.GREEDY_SEARCH_LAUNCH_LOCK_FILE||`${H}/browser-launch.lock`,bd=process.env.GREEDY_SEARCH_ACTIVITY_FILE||`${H}/browser-last-activity`,vd=(process.env.CDP_SOCKET_DIR||`${H}/cdp-sockets`).replaceAll("\\","/"),$d=`${H}/visible-recovery.jsonl`,Ot=vn(el(),".dm"),B=vn(Ot,"greedyconfig"),Nt=["perplexity","google","chatgpt","gemini"],Pt="gemini";function il(){try{if(Le(B)){let e=bn(B,"utf8"),t=JSON.parse(e);if(Array.isArray(t.engines)&&t.engines.length>0&&t.engines.every((r)=>typeof r==="string")){let r=t.engines.filter((i)=>$t[i]),n=t.engines.filter((i)=>!$t[i]);if(n.length>0)process.stderr.write(`[greedysearch] Warning: ignoring unknown engine(s) in ${B}: ${n.join(", ")}
|
|
783
783
|
[greedysearch] Available engines: ${Object.keys($t).join(", ")}
|
|
784
784
|
`);if(r.length>0)return r;process.stderr.write(`[greedysearch] Warning: no valid engines in ${B}, falling back to defaults: ${Nt.join(", ")}
|
|
785
|
-
`)}}}catch{}return Nt}function
|
|
786
|
-
`,"utf8")}catch{}}
|
|
785
|
+
`)}}}catch{}return Nt}function al(){try{if(!Le(Ot))Zs(Ot,{recursive:!0});if(!Le(B))Xs(B,JSON.stringify({engines:Nt,synthesizer:Pt},null,2)+`
|
|
786
|
+
`,"utf8")}catch{}}al();var en=["gemini","chatgpt"];function ol(){try{if(Le(B)){let e=bn(B,"utf8"),t=JSON.parse(e);if(typeof t.synthesizer==="string"){let r=t.synthesizer.toLowerCase();if(en.includes(r))return r;process.stderr.write(`[greedysearch] Warning: unknown synthesizer "${t.synthesizer}" in ${B}
|
|
787
787
|
[greedysearch] Available synthesizers: ${en.join(", ")}
|
|
788
|
-
[greedysearch] Falling back to default: ${
|
|
789
|
-
`)}}}catch{}return
|
|
790
|
-
|
|
791
|
-
${i}`,pages:n.total}}catch(r){return{error:r.message||String(r)}}}function $n(
|
|
792
|
-
`)}}if(
|
|
793
|
-
`);let i=await
|
|
794
|
-
`)}let n=await
|
|
795
|
-
`),await
|
|
788
|
+
[greedysearch] Falling back to default: ${Pt}
|
|
789
|
+
`)}}}catch{}return Pt}var $t={perplexity:"perplexity.mjs",p:"perplexity.mjs",bing:"bing-copilot.mjs",b:"bing-copilot.mjs",google:"google-ai.mjs",g:"google-ai.mjs",gemini:"gemini.mjs",gem:"gemini.mjs",chatgpt:"chatgpt.mjs",gpt:"chatgpt.mjs","semantic-scholar":"semantic-scholar.mjs",semanticscholar:"semantic-scholar.mjs",s2:"semantic-scholar.mjs",logically:"logically.mjs",log:"logically.mjs"},_d=il(),Ed=ol(),tn=Math.max(1,Number.parseInt(process.env.GREEDY_FETCH_CONCURRENCY||"5",10)||5);process.env.CDP_PROFILE_DIR=H;function sl(){if(typeof globalThis.DOMMatrix>"u")globalThis.DOMMatrix=class{constructor(e=void 0){}multiplySelf(){return this}preMultiplySelf(){return this}translateSelf(){return this}scaleSelf(){return this}rotateSelf(){return this}};if(typeof globalThis.ImageData>"u")globalThis.ImageData=class{constructor(e=void 0,t=0,r=0){this.data=e,this.width=t,this.height=r}};if(typeof globalThis.Path2D>"u")globalThis.Path2D=class{constructor(e=void 0){}}}async function ll(){sl();let e=await import("pdf-parse"),t=e.PDFParse??e.default;if(!t)throw Error("pdf-parse did not export PDFParse");return t}async function cl(e,t){try{let r=new(await ll())({data:new Uint8Array(e)});await r.load();let n=await r.getText(),i=n.text?.trim();if(!i)return null;return{title:new URL(t).pathname.split("/").pop()||"Document.pdf",content:`## PDF Content (${n.total} pages)
|
|
790
|
+
|
|
791
|
+
${i}`,pages:n.total}}catch(r){return{error:r.message||String(r)}}}function $n(e="",t=240){let r=String(e).replaceAll(/\s+/g," ").trim();if(r.length<=t)return r;let n=r.slice(0,t),i=n.lastIndexOf(" ");return i>0?`${n.slice(0,i)}...`:`${n}...`}async function ul(e,t,r=8000){let n=Date.now();try{let i=(await D(["evalraw",e,"Page.getFrameTree","{}"]).then((p)=>JSON.parse(p)).catch(()=>null))?.frameTree?.frame?.id||void 0,a=await D(["evalraw",e,"Network.loadNetworkResource",JSON.stringify({frameId:i,url:t,options:{disableCache:!0,includeCredentials:!1}})],20000),o=JSON.parse(a).resource;if(!o?.success||!o.httpStatusCode)return{url:t,error:o?.netErrorName||o?.netError||"loadNetworkResource failed",source:"chrome",duration:Date.now()-n,needsFallback:!0};let s="";if(o.stream)try{let p=await D(["evalraw",e,"IO.read",JSON.stringify({handle:o.stream})],1e4);s=JSON.parse(p).data||"",await D(["evalraw",e,"IO.close",JSON.stringify({handle:o.stream})]).catch(()=>{})}catch{}if(!s||s.length<100)return{url:t,error:"Empty response body from Network.loadNetworkResource",source:"chrome",duration:Date.now()-n,needsFallback:!0};let l=on(o.httpStatusCode,s,t,t);if(l.blocked)return{url:t,status:o.httpStatusCode,error:`Blocked: ${l.reason}`,source:"chrome",duration:Date.now()-n,needsBrowser:!0};let c=await sn(s,t),u=ln(c);if(!u.ok)return{url:t,status:o.httpStatusCode,error:`Low quality: ${u.reason}`,source:"chrome",duration:Date.now()-n,needsBrowser:!0};let d=ne(c.markdown,r);return{url:t,finalUrl:t,status:o.httpStatusCode,contentType:"text/markdown",lastModified:"",publishedTime:c.publishedTime||"",byline:c.byline||"",siteName:c.siteName||"",lang:c.lang||"",title:c.title||t,snippet:c.excerpt,content:d,contentChars:d.length,source:"chrome",duration:Date.now()-n}}catch(i){return{url:t,error:i.message,source:"chrome",duration:Date.now()-n,needsFallback:!0}}}function _n(e){try{return new URL(e).pathname.toLowerCase().endsWith(".pdf")}catch{return!1}}async function dl(e,t=8000){let r=_t(e);if(r.blocked)return{url:e,finalUrl:e,status:403,error:`Blocked: ${r.reason}`,source:"pdf-http"};let n=new AbortController,i=setTimeout(()=>n.abort(),20000),a=Date.now();try{let o=await fetch(e,{method:"GET",redirect:"follow",signal:n.signal,headers:os({accept:"application/pdf,application/octet-stream;q=0.9,*/*;q=0.5"})});clearTimeout(i);let s=o.headers.get("content-type")||"",l=o.url||e,c=Number.parseInt(o.headers.get("content-length")||"0",10);if(o.status>=400)return{url:e,finalUrl:l,status:o.status,error:`HTTP ${o.status}`,source:"pdf-http",duration:Date.now()-a};if(!s.toLowerCase().includes("application/pdf")&&!_n(l))return null;if(c>31457280)return{url:e,finalUrl:l,status:o.status,error:`PDF too large: ${c} bytes`,source:"pdf-http",duration:Date.now()-a};let u=Buffer.from(await o.arrayBuffer()),d=await cl(u,l);if(!d||d.error)return{url:e,finalUrl:l,status:o.status,error:d?.error||"PDF text extraction failed",source:"pdf-http",duration:Date.now()-a};let p=ne(d.content,t);return{url:e,finalUrl:l,status:o.status,contentType:"application/pdf",lastModified:o.headers.get("last-modified")||"",title:d.title,snippet:$n(p,320),content:p,contentChars:p.length,pages:d.pages,source:"pdf-http",duration:Date.now()-a}}catch(o){return clearTimeout(i),{url:e,finalUrl:e,error:o.message||String(o),source:"pdf-http",duration:Date.now()-a}}}async function fl(e,t=8000){let r=Date.now();if(_n(e)){let i=await dl(e,t);if(i?.content||i?.status===403)return i}if(Et(e)){let i=Et(e);if(i&&(i.type==="root"||i.type==="tree"||i.type==="blob"&&!i.path?.includes("."))){let a=await ms(e);if(a.ok){let o=ne(a.content,t);return{url:e,finalUrl:e,status:200,contentType:"text/markdown",lastModified:"",title:a.title,snippet:o.slice(0,320),content:o,contentChars:o.length,source:"github-api",...a.tree&&{tree:a.tree},duration:Date.now()-r}}process.stderr.write(`[greedysearch] GitHub API fetch failed, trying HTTP: ${a.error}
|
|
792
|
+
`)}}if(ws(e)?.type==="post"){process.stderr.write(`[greedysearch] Using Reddit JSON API for: ${e.slice(0,60)}...
|
|
793
|
+
`);let i=await bs(e,t);if(i.ok){let a=ne(i.markdown,t);return{url:e,finalUrl:i.finalUrl,status:i.status,contentType:"text/markdown",lastModified:i.lastModified||"",publishedTime:i.publishedTime||"",byline:i.byline||"",siteName:i.siteName||"",lang:i.lang||"",title:i.title,snippet:i.excerpt,content:a,contentChars:a.length,source:"reddit-api",duration:Date.now()-r}}process.stderr.write(`[greedysearch] Reddit API fetch failed, falling back to HTTP: ${i.error}
|
|
794
|
+
`)}let n=await ls(e,{timeoutMs:1e4});if(n.ok){let i=ne(n.markdown,t);return{url:e,finalUrl:n.finalUrl,status:n.status,contentType:"text/markdown",lastModified:n.lastModified||"",publishedTime:n.publishedTime||"",byline:n.byline||"",siteName:n.siteName||"",lang:n.lang||"",title:n.title,snippet:n.excerpt,content:i,contentChars:i.length,source:"http",duration:Date.now()-r}}if(n.needsBrowser)try{let i=await yn();try{let a=await ul(i,e,t);if(a.content&&a.content.length>100)return a}finally{await wn(i)}}catch{}return process.stderr.write(`[greedysearch] HTTP failed for ${e.slice(0,60)}, trying browser...
|
|
795
|
+
`),await hl(e,t)}async function pl(e,t=4000,r=200){let n=Date.now()+t;while(Date.now()<n){try{if((await D(["eval",e,'document.readyState === "complete" && !!document.body && document.body.innerText.length > 500'])).trim()==="true")return}catch{}await new Promise((i)=>setTimeout(i,r))}}async function hl(e,t=8000){let r=Date.now(),n;try{n=await yn()}catch(i){return{url:e,title:"",content:null,snippet:"",contentChars:0,error:`openNewTab failed: ${i.message}`,source:"browser",duration:Date.now()-r}}try{await D(["nav",n,e],30000),await pl(n);let i=await D(["eval",n,String.raw`
|
|
796
796
|
(function(){
|
|
797
797
|
var el = document.querySelector('article, [role="main"], main, .post-content, .article-body, #content, .content');
|
|
798
798
|
var text = (el || document.body).innerText;
|
|
@@ -802,26 +802,26 @@ ${i}`,pages:n.total}}catch(r){return{error:r.message||String(r)}}}function $n(t=
|
|
|
802
802
|
url: location.href
|
|
803
803
|
});
|
|
804
804
|
})()
|
|
805
|
-
`]),a=JSON.parse(i),o=ne(a.content,
|
|
806
|
-
`);let o=Array(i.length),s=0,l=0;async function c(){while(!0){let f=s++;if(f>=i.length)return;let m=i[f],
|
|
807
|
-
`);let
|
|
808
|
-
`);else if(
|
|
805
|
+
`]),a=JSON.parse(i),o=ne(a.content,t);return{url:e,finalUrl:a.url||e,status:200,contentType:"text/plain",lastModified:"",title:a.title,snippet:$n(o,320),content:o,contentChars:o.length,source:"browser",duration:Date.now()-r}}catch(i){return{url:e,title:"",content:null,snippet:"",contentChars:0,error:i.message,source:"browser",duration:Date.now()-r}}finally{await wn(n)}}async function En(e,t=5,r=8000,n=tn){let i=e.slice(0,t);if(i.length===0)return[];let a=Math.min(i.length,Math.max(1,Number.parseInt(String(n),10)||tn));process.stderr.write(`[greedysearch] Fetching content from ${i.length} sources via HTTP (concurrency ${a})...
|
|
806
|
+
`);let o=Array(i.length),s=0,l=0;async function c(){while(!0){let f=s++;if(f>=i.length)return;let m=i[f],h=m.canonicalUrl||m.url;process.stderr.write(`[greedysearch] [${f+1}/${i.length}] Fetching: ${h.slice(0,60)}...
|
|
807
|
+
`);let g=await fl(h,r).catch((y)=>({url:h,title:"",content:null,snippet:"",contentChars:0,error:y.message,source:"error",duration:0}));if(o[f]={id:m.id,...g},g.content&&g.content.length>100)process.stderr.write(`[greedysearch] ✓ ${g.source}: ${g.content.length} chars
|
|
808
|
+
`);else if(g.error)process.stderr.write(`[greedysearch] ✗ ${g.error.slice(0,80)}
|
|
809
809
|
`);l+=1,process.stderr.write(`PROGRESS:fetch:${l}/${i.length}
|
|
810
810
|
`)}}await Promise.all(Array.from({length:a},()=>c()));let u=o.filter((f)=>f.content&&f.content.length>100),d=o.filter((f)=>f.source==="http").length,p=o.filter((f)=>f.source==="browser").length;return process.stderr.write(`[greedysearch] Fetched ${u.length}/${o.length} sources (HTTP: ${d}, Browser: ${p})
|
|
811
|
-
`),o}import{spawn as
|
|
812
|
-
`)}return{startRound(y){o=y-1},endRound(){o++,
|
|
813
|
-
`);c=p.pop()||"";for(let f of p)if(
|
|
814
|
-
`)});let u=setTimeout(()=>{o.kill(),a(Error(`research child search timed out for: ${
|
|
815
|
-
`);let p=An({totalActions:6,totalRounds:1,totalFetches:1,silent:process.env.GREEDY_RESEARCH_QUIET==="1"});p.startRound(1);let f=[],m=[],
|
|
811
|
+
`),o}import{spawn as bl}from"node:child_process";import{basename as gl}from"node:path";function Me(e=process.env,t=process.execPath){let r=e.GREEDY_SEARCH_NODE||e.NODE_BINARY||e.NODE;if(r?.trim())return r.trim();let n=gl(t||"").toLowerCase();if(n==="node"||n==="node.exe")return t;return"node"}import{existsSync as Sn}from"node:fs";import{dirname as ml,join as Ue}from"node:path";import{fileURLToPath as yl}from"node:url";function Rn(e){return yl(new URL(".",e)).replace(/^\/([A-Z]:)/,"$1")}function wl(e,t=Sn){let r=new Set;for(let n of e.filter(Boolean)){if(r.has(n))continue;if(r.add(n),t(n))return n}return e.filter(Boolean).at(-1)}function Dn(e,{moduleDir:t,entrypoint:r=process.argv[1],env:n=process.env,exists:i=Sn}={}){let a=r?ml(r):null,o=n.GREEDY_SEARCH_EXTENSION_DIR?.trim()||null;return wl([t?Ue(t,"..","..","extractors",e):null,t?Ue(t,"..","extractors",e):null,o?Ue(o,"extractors",e):null,a?Ue(a,"..","extractors",e):null],i)}var vl=Rn(import.meta.url),$l={gemini:"gemini.mjs",chatgpt:"chatgpt.mjs"};function _l(e="gemini"){let t=String(e||"gemini").toLowerCase();if(t==="gem")return"gemini";if(t==="gpt")return"chatgpt";return t}async function El(e,t,{tabPrefix:r=null,timeoutMs:n=180000,visible:i=null}={}){let a=_l(e),o=$l[a];if(!o||!ae.includes(a))throw Error(`Unsupported synthesizer "${e}". Supported: ${ae.join(", ")}`);return new Promise((s,l)=>{let c=Dn(o,{moduleDir:vl}),u=r?["--tab",String(r)]:[],d={...process.env,CDP_PROFILE_DIR:k};if(i!==!0)delete d.GREEDY_SEARCH_VISIBLE,delete d.GREEDY_SEARCH_ALWAYS_VISIBLE;else d.GREEDY_SEARCH_VISIBLE="1",d.GREEDY_SEARCH_ALWAYS_VISIBLE="1";let p=bl(Me(),[c,"--stdin",...u],{stdio:["pipe","pipe","pipe"],env:d});p.stdin.write(t),p.stdin.end();let f="",m="";p.stdout.on("data",(g)=>f+=g),p.stderr.on("data",(g)=>m+=g);let h=setTimeout(()=>{p.kill(),l(Error(`${a} prompt timed out after ${n/1000}s`))},n);p.on("close",(g)=>{if(clearTimeout(h),g!==0){l(Error(m.trim()||`${a} extractor failed`));return}try{s(JSON.parse(f.trim()))}catch{l(Error(`bad JSON from ${a}: ${f.slice(0,100)}`))}})})}async function xt(e,t={}){return El("gemini",e,t)}function Sl(e){if(e<1000)return"0s";let t=Math.round(e/1000);if(t<60)return`${t}s`;let r=Math.floor(t/60),n=t%60;return`${r}m ${n}s`}function Rl(e,t=20){let r=Math.round(e*t),n=t-r;return"["+"█".repeat(r)+"░".repeat(n)+"]"}function An({totalActions:e=0,totalRounds:t=0,totalFetches:r=0,silent:n=!1}={}){let i=Date.now(),a=0,o=0,s=0,l=[],c=null,u=null,d=0,p=500;function f(y){if(l.push(y),l.length>5)l.shift()}function m(){if(l.length===0)return null;return l.reduce((y,v)=>y+v,0)/l.length}function h(y){let v=Date.now()-i,b=e+r+t,w=a+s+o,E=b>0?Math.min(1,w/b):0,A=Rl(E),S=m(),Ge=Math.max(0,b-w),le=S?S*Ge:null,ce=le?Sl(le):"—",Z=u?` ${u}`:"";return`${A} ${w}/${b} (${y}${Z}, ETA ${ce})`}function g(y){if(n)return;let v=Date.now();if(v-d<p&&y!=="done")return;d=v,process.stderr.write(`[greedysearch] ${h(y)}
|
|
812
|
+
`)}return{startRound(y){o=y-1},endRound(){o++,g("round")},startAction(y,v){c=Date.now(),u=`${y}:${(v||"").slice(0,40)}`,g(y)},endAction(){if(c)f(Date.now()-c),c=null;a++,g("action")},startFetch(y){c=Date.now(),u=`fetch:${(y||"").slice(0,40)}`,g("fetch")},endFetch(y=!0){if(c)f(Date.now()-c),c=null;s++,g(y?"fetch":"fetch-failed")},print(){g("progress")},finish(){g("done")},getElapsedMs(){return Date.now()-i}}}import{spawn as Dl}from"node:child_process";import{join as Al}from"node:path";import{fileURLToPath as Cl}from"node:url";var kl=Cl(new URL(".",import.meta.url)).replace(/^\/([A-Z]:)/,"$1"),Ol=Al(kl,"..","..","bin","search.mjs");function Cn(e,t=1/0){let r=new Set,n=[];for(let i of e||[]){let a=te(String(i||""),1000);if(!a||r.has(a))continue;if(r.add(a),n.push(a),n.length>=t)break}return n}function Nl(e){let t=String(e||"").trim();if(!t)return[];return[`${t} — definition and overview`,`${t} — how it works, mechanism, or key details`,`${t} — current usage, comparison, or best practices`]}function Pl(e,t){let r=new Map;for(let n of e||[]){let i=n?.canonicalUrl||n?.finalUrl||n?.url;if(i)r.set(i,n)}for(let n of t||[]){let i=n?.canonicalUrl||n?.finalUrl||n?.url;if(!i)continue;if(r.has(i)){let a={...r.get(i),angles:[...r.get(i).angles||[r.get(i).query||""],n.query||""]};r.set(i,a)}else r.set(i,n)}return Array.from(r.values())}var xl=pe.join("|"),Il=new RegExp(`^\\[(${xl})\\]`);function jl(e){return/^PROGRESS:/.test(e)||/^\[greedysearch\]/.test(e)||Il.test(e)||/^GreedySearch Chrome/.test(e)||/^Launching GreedySearch Chrome/.test(e)||/^Headless mode/.test(e)||/^Ready\.?$/.test(e)}async function Tl(e,{locale:t=null,short:r=!0}={}){let n=[Ol,"all","--inline","--stdin","--fast"];if(!r)n.push("--full");if(t)n.push("--locale",t);return new Promise((i,a)=>{let o=Dl(Me(),n,{stdio:["pipe","pipe","pipe"],env:{...process.env,GREEDY_SEARCH_RESEARCH_CHILD:"1"}});o.stdin.write(e),o.stdin.end();let s="",l="",c="";o.stdout.on("data",(d)=>s+=d),o.stderr.on("data",(d)=>{l+=d,c+=d.toString();let p=c.split(`
|
|
813
|
+
`);c=p.pop()||"";for(let f of p)if(jl(f))process.stderr.write(`${f}
|
|
814
|
+
`)});let u=setTimeout(()=>{o.kill(),a(Error(`research child search timed out for: ${e}`))},140000);o.on("close",(d)=>{if(clearTimeout(u),d!==0){a(Error(l.trim()||`search child exited with code ${d}`));return}try{i(JSON.parse(s.trim()))}catch{a(Error(`Invalid JSON from research child: ${s.slice(0,200)}`))}})})}function Ll(e,t){let r=new Map;for(let n of t||[]){let i=ge(n?.canonicalUrl||n?.finalUrl||n?.url);if(i&&n?.id)r.set(i,n.id)}return(e||[]).map((n,i)=>{let a=ge(n?.finalUrl||n?.canonicalUrl||n?.url);return{...n,id:n?.id||r.get(a)||`F${i+1}`}})}function Ml(e){let t=e.length,r=e.filter((n)=>n.status==="closed").length;return{total:t,closed:r,open:Math.max(0,t-r)}}async function rf({query:e,locale:t=null,maxSources:r=5,qualityThreshold:n=8.5,writeBundle:i=process.env.GREEDY_RESEARCH_BUNDLE!=="0",researchOutDir:a=null}={}){let o=new Date().toISOString(),s=Date.now(),l=Hr(e),c=new Set;process.stderr.write(`[greedysearch] Simple research mode: single-pass for "${te(e,80)}"
|
|
815
|
+
`);let p=An({totalActions:6,totalRounds:1,totalFetches:1,silent:process.env.GREEDY_RESEARCH_QUIET==="1"});p.startRound(1);let f=[],m=[],h=Nl(e),g=[];p.startAction("search",`${h.length} angles in parallel`);let y=await Promise.allSettled(h.map(($)=>Tl($,{locale:t,short:!0})));for(let $=0;$<h.length;$++){let Q=h[$],ie=y[$];if(p.endAction(),ie.status==="fulfilled"){let de=ie.value;g.push({angle:Q,result:de});let K=Be(de,Q);f=Pl(f,K)}else process.stderr.write(`[greedysearch] Simple search angle "${Q}" failed: ${ie.reason.message}
|
|
816
816
|
`)}if(process.stderr.write(`PROGRESS:research:simple:fetching
|
|
817
817
|
`),f.length>0)try{p.startFetch(`top ${Math.min(r,f.length)} sources`),m=await En(f,Math.min(r,f.length),8000,Math.min(3,r)),p.endFetch(!0),f=Ut(f,m)}catch($){p.endFetch(!1),process.stderr.write(`[greedysearch] Source fetching failed: ${$.message}
|
|
818
|
-
`)}m=
|
|
819
|
-
`);let v=[];try{let $=await Tr({query:
|
|
818
|
+
`)}m=Ll(m,f),process.stderr.write(`PROGRESS:research:simple:evidence
|
|
819
|
+
`);let v=[];try{let $=await Tr({query:e,questions:l,fetchedSources:m,extractedSourceKeys:c});v=$.evidence||[];for(let Q of $.evidence){let ie=Array.isArray(Q.answers)?Q.answers:[];for(let K of ie){let ee=K?.id||K?.question;if(ee){let X=l.find((kn)=>kn.id===ee);if(X){if(X.status="closed",X.closedRound=1,K.evidence)X.evidence=Cn([...X.evidence||[],K.evidence],4)}}}let de=Array.isArray(Q.newQuestions)?Q.newQuestions:[];for(let K of de){let ee=te(String(K),320);if(ee&&!l.some((X)=>X.question===ee))l.push({id:`Q${l.length+1}`,question:ee,status:"open",reason:"Discovered gap/follow-up",createdRound:1,evidence:[],sourceIds:[]})}}}catch($){process.stderr.write(`[greedysearch] Evidence extraction failed: ${$.message}
|
|
820
820
|
`)}process.stderr.write(`PROGRESS:research:simple:synthesizing
|
|
821
|
-
`);let b={answer:"",agreement:{level:"mixed",summary:"Single-pass synthesis."},differences:[],caveats:[],claims:[],recommendedSources:f.slice(0,4).map(($)=>$.id),synthesized:!1};if(v.length>0)try{p.startAction("synth-evidence","from evidence");let $=await
|
|
822
|
-
`)}if(!b.synthesized&&f.length>0)try{p.startAction("synth-final","fallback report");let $=await
|
|
821
|
+
`);let b={answer:"",agreement:{level:"mixed",summary:"Single-pass synthesis."},differences:[],caveats:[],claims:[],recommendedSources:f.slice(0,4).map(($)=>$.id),synthesized:!1};if(v.length>0)try{p.startAction("synth-evidence","from evidence");let $=await xt(Mr(e,f,l,v),{timeoutMs:120000});p.endAction(),b={...b,...xe($?.answer||"")||{}},b.synthesized=Array.isArray(b.claims)&&b.claims.length>0}catch($){process.stderr.write(`[greedysearch] Evidence synthesis failed: ${$.message}
|
|
822
|
+
`)}if(!b.synthesized&&f.length>0)try{p.startAction("synth-final","fallback report");let $=await xt(Lr(e,[{round:1,learnings:[],gaps:[],actions:[]}],f,l,v),{timeoutMs:120000});p.endAction(),b={...b,...xe($?.answer||"")||{}},b.synthesized=Array.isArray(b.claims)&&b.claims.length>0}catch($){process.stderr.write(`[greedysearch] Final synthesis failed: ${$.message}
|
|
823
823
|
`)}process.stderr.write(`PROGRESS:research:simple:audit
|
|
824
824
|
`);let w=Ur(b.answer||"",f),E=await Gr(f,w);qr(l,b,w);let A=Cn(b.caveats||[]),S=Fr({sources:f,fetchedSources:m,synthesis:b,citationAudit:w,gaps:A,questions:l,rounds:[{round:1,actions:[],learnings:[],gaps:A}],qualityScore:b.synthesized?8:5,qualityThreshold:n,maxSources:r}),Ge=new Date().toISOString(),le=Date.now()-s,ce={startedAt:o,finishedAt:Ge,durationMs:le,rounds:1,terminationReason:"simple_single_pass"},Z=null,ue;if(i){process.stderr.write(`PROGRESS:research:simple:bundle
|
|
825
|
-
`);try{Z=await Jr({query:
|
|
825
|
+
`);try{Z=await Jr({query:e,rounds:[{round:1,actions:[],learnings:[],gaps:A,evidence:v}],sources:f,fetchedSources:m,evidenceItems:v,synthesis:b,citationAudit:w,citationUrls:E,floor:S,manifest:{...ce,engines:Tt,synthesizer:"gemini",actionsRun:1,searches:1,fetches:m.length,sourcesFetched:m.filter(($)=>$?.contentChars>100).length,engineFailures:[],floorMet:S.floorMet},allGaps:A,questions:l,outDir:a}),ue=Z.sourceFiles,delete Z.sourceFiles}catch($){process.stderr.write(`[greedysearch] Research bundle write failed: ${$.message}
|
|
826
826
|
`),Z={error:$.message||String($)},ue=await wt(m)}}else ue=await wt(m);return process.stderr.write(`PROGRESS:research:done
|
|
827
|
-
`),p.endRound(),p.finish(),{query:
|
|
827
|
+
`),p.endRound(),p.finish(),{query:e,_research:{mode:"simple",breadth:1,iterations:1,maxSources:r,rounds:[{round:1,actions:[],learnings:[],gaps:A,evidence:v}],learnings:[],gaps:A,evidence:v,questions:l,questionProgress:Ml(l),qualityHistory:[b.synthesized?8:5],terminationReason:"simple_single_pass",qualityThreshold:n,floor:S,bundle:Z,manifest:ce},_citationAudit:w,_citationUrls:E,_sources:f,_fetchedSources:ue,_synthesis:b,_confidence:{sourcesCount:f.length,fetchedSourceSuccessRate:m.length>0?m.filter(($)=>$.contentChars>100).length/m.length:0,agreementLevel:b.agreement?.level||"mixed",floorMet:S.floorMet}}}export{rf as runSimpleResearchMode,Pl as mergeSourcesByUrl,Nl as buildSearchAngles};
|