@duckmind/dm-darwin-x64 0.52.6 → 0.53.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dm +0 -0
- package/extensions/.dm-extensions.json +14 -10
- package/extensions/dm-cua/bin/browser-cua.mjs +6 -6
- package/extensions/dm-cua/index.js +5 -5
- package/extensions/dm-cua/src/browser-cua-lib.mjs +2 -2
- package/extensions/greedysearch-dm/bin/cdp-greedy.mjs +8 -1
- package/extensions/greedysearch-dm/bin/cdp-headless.mjs +1 -1
- package/extensions/greedysearch-dm/bin/cdp-visible.mjs +1 -1
- package/extensions/greedysearch-dm/bin/cdp.mjs +29 -20
- package/extensions/greedysearch-dm/bin/gschrome.mjs +1 -8
- package/extensions/greedysearch-dm/bin/kill-visible.mjs +1 -1
- package/extensions/greedysearch-dm/bin/launch-visible.mjs +1 -2
- package/extensions/greedysearch-dm/bin/launch.mjs +9 -11
- package/extensions/greedysearch-dm/bin/mcp.mjs +375 -0
- package/extensions/greedysearch-dm/bin/search.mjs +309 -157
- package/extensions/greedysearch-dm/bin/visible.mjs +1 -1
- package/extensions/greedysearch-dm/extractors/bing-copilot.mjs +205 -68
- package/extensions/greedysearch-dm/extractors/chatgpt.mjs +229 -140
- package/extensions/greedysearch-dm/extractors/common.mjs +160 -50
- package/extensions/greedysearch-dm/extractors/consensus.mjs +226 -93
- package/extensions/greedysearch-dm/extractors/consent.mjs +44 -17
- package/extensions/greedysearch-dm/extractors/gemini.mjs +277 -138
- package/extensions/greedysearch-dm/extractors/google-ai.mjs +172 -64
- package/extensions/greedysearch-dm/extractors/logically.mjs +183 -69
- package/extensions/greedysearch-dm/extractors/perplexity.mjs +331 -65
- package/extensions/greedysearch-dm/extractors/semantic-scholar.mjs +118 -41
- package/extensions/greedysearch-dm/index.js +23 -20
- package/extensions/greedysearch-dm/package.json +8 -3
- package/extensions/greedysearch-dm/skills/greedy-search/skill.md +8 -8
- package/extensions/greedysearch-dm/src/fetcher.mjs +2 -2
- package/extensions/greedysearch-dm/src/formatters/results.js +8 -6
- package/extensions/greedysearch-dm/src/github.mjs +7 -7
- package/extensions/greedysearch-dm/src/reddit.mjs +1 -1
- package/extensions/greedysearch-dm/src/search/browser-lifecycle.mjs +10 -8
- package/extensions/greedysearch-dm/src/search/challenge-detect.mjs +48 -0
- package/extensions/greedysearch-dm/src/search/chrome.mjs +130 -54
- package/extensions/greedysearch-dm/src/search/constants.mjs +6 -6
- package/extensions/greedysearch-dm/src/search/engines.mjs +9 -9
- package/extensions/greedysearch-dm/src/search/fetch-source.mjs +160 -83
- package/extensions/greedysearch-dm/src/search/minimize.mjs +1 -0
- package/extensions/greedysearch-dm/src/search/paths.mjs +1 -1
- package/extensions/greedysearch-dm/src/search/pdf.mjs +1 -1
- package/extensions/greedysearch-dm/src/search/port-pid.mjs +1 -1
- package/extensions/greedysearch-dm/src/search/progress.mjs +2 -0
- package/extensions/greedysearch-dm/src/search/recovery.mjs +1 -1
- package/extensions/greedysearch-dm/src/search/research.mjs +262 -167
- package/extensions/greedysearch-dm/src/search/scale-aware.mjs +11 -0
- package/extensions/greedysearch-dm/src/search/simple-research.mjs +827 -0
- package/extensions/greedysearch-dm/src/search/synthesis-runner.mjs +9 -9
- package/extensions/greedysearch-dm/src/search/synthesis.mjs +10 -10
- package/extensions/greedysearch-dm/src/tools/greedy-search-handler.js +23 -20
- package/extensions/greedysearch-dm/src/tools/shared.js +10 -9
- package/extensions/greedysearch-dm/src/utils/node-runtime.mjs +1 -1
- package/package.json +1 -1
- package/extensions/greedysearch-dm/src/search/cdp-endpoint.mjs +0 -3
- package/extensions/greedysearch-dm/src/search/launcher-paths.mjs +0 -3
- package/extensions/greedysearch-dm/src/search/partial-output.mjs +0 -1
- package/extensions/greedysearch-dm/src/utils/browser-executable.mjs +0 -1
|
@@ -0,0 +1,827 @@
|
|
|
1
|
+
import{createRequire as On}from"node:module";var K=On(import.meta.url);import{existsSync as fe,mkdirSync as Nn,readFileSync as It,writeFileSync as xn}from"node:fs";import{homedir as Pn}from"node:os";import{join as jt}from"node:path";import{tmpdir as In}from"node:os";function jn(t,e=9222){let r=Number.parseInt(String(t??""),10);return Number.isInteger(r)&&r>1024&&r<65535?r:e}var Tn=In().replaceAll("\\","/"),Fl=jn(process.env.GREEDY_SEARCH_PORT),k=(process.env.GREEDY_SEARCH_PROFILE_DIR||process.env.CDP_PROFILE_DIR||`${Tn}/greedysearch-chrome-profile`).replaceAll("\\","/"),Hl=`${k}/DevToolsActivePort`,ql=process.env.GREEDY_SEARCH_PID_FILE||`${k}/browser.pid`,Jl=process.env.CDP_PAGES_CACHE||`${k}/cdp-pages.json`,Yl=process.env.GREEDY_SEARCH_MODE_FILE||`${k}/browser-mode`,Wl=process.env.GREEDY_SEARCH_METADATA_FILE||`${k}/browser-metadata.json`,Bl=process.env.GREEDY_SEARCH_LAUNCH_LOCK_FILE||`${k}/browser-launch.lock`,Ql=process.env.GREEDY_SEARCH_ACTIVITY_FILE||`${k}/browser-last-activity`,Vl=(process.env.CDP_SOCKET_DIR||`${k}/cdp-sockets`).replaceAll("\\","/"),Kl=`${k}/visible-recovery.jsonl`,He=jt(Pn(),".dm"),q=jt(He,"greedyconfig"),qe=["perplexity","google","chatgpt","gemini"],Je="gemini";function Ln(){try{if(fe(q)){let t=It(q,"utf8"),e=JSON.parse(t);if(Array.isArray(e.engines)&&e.engines.length>0&&e.engines.every((r)=>typeof r==="string")){let r=e.engines.filter((i)=>Fe[i]),n=e.engines.filter((i)=>!Fe[i]);if(n.length>0)process.stderr.write(`[greedysearch] Warning: ignoring unknown engine(s) in ${q}: ${n.join(", ")}
|
|
2
|
+
[greedysearch] Available engines: ${Object.keys(Fe).join(", ")}
|
|
3
|
+
`);if(r.length>0)return r;process.stderr.write(`[greedysearch] Warning: no valid engines in ${q}, falling back to defaults: ${qe.join(", ")}
|
|
4
|
+
`)}}}catch{}return qe}function Mn(){try{if(!fe(He))Nn(He,{recursive:!0});if(!fe(q))xn(q,JSON.stringify({engines:qe,synthesizer:Je},null,2)+`
|
|
5
|
+
`,"utf8")}catch{}}Mn();var ae=["gemini","chatgpt"];function Un(){try{if(fe(q)){let t=It(q,"utf8"),e=JSON.parse(t);if(typeof e.synthesizer==="string"){let r=e.synthesizer.toLowerCase();if(ae.includes(r))return r;process.stderr.write(`[greedysearch] Warning: unknown synthesizer "${e.synthesizer}" in ${q}
|
|
6
|
+
[greedysearch] Available synthesizers: ${ae.join(", ")}
|
|
7
|
+
[greedysearch] Falling back to default: ${Je}
|
|
8
|
+
`)}}}catch{}return Je}var Fe={perplexity:"perplexity.mjs",p:"perplexity.mjs",bing:"bing-copilot.mjs",b:"bing-copilot.mjs",google:"google-ai.mjs",g:"google-ai.mjs",gemini:"gemini.mjs",gem:"gemini.mjs",chatgpt:"chatgpt.mjs",gpt:"chatgpt.mjs","semantic-scholar":"semantic-scholar.mjs",semanticscholar:"semantic-scholar.mjs",s2:"semantic-scholar.mjs",logically:"logically.mjs",log:"logically.mjs"},pe=Ln(),Tt=pe,zl=Un(),Zl=Math.max(1,Number.parseInt(process.env.GREEDY_FETCH_CONCURRENCY||"5",10)||5);process.env.CDP_PROFILE_DIR=k;var Gn=["fbclid","gclid","ref","ref_src","ref_url","source","utm_campaign","utm_content","utm_medium","utm_source","utm_term"],Fn=["dev.to","hashnode.com","medium.com","reddit.com","stackoverflow.com","stackexchange.com","substack.com"],Hn=["arstechnica.com","techcrunch.com","theverge.com","venturebeat.com","wired.com","zdnet.com"],Mt=["facebook.com","instagram.com","linkedin.com","pinterest.com","tiktok.com","twitter.com","x.com"];function te(t="",e=240){let r=String(t).replaceAll(/\s+/g," ").trim();if(r.length<=e)return r;let n=r.slice(0,e),i=n.lastIndexOf(" ");return i>0?`${n.slice(0,i)}...`:`${n}...`}function Ye(t=""){let e=te(t,180);if(!e)return"";if(/^https?:\/\//i.test(e))return"";let r=e.split(/\s+/).filter(Boolean).length,n=/[A-Z]/.test(e),i=/\d/.test(e);return e===e.toLowerCase()&&r<=4&&!n&&!i?"":e}function We(t="",e=""){let r=Ye(t),n=Ye(e);if(!n)return r;if(!r)return n;let i=/^https?:\/\//i.test(r),a=/^https?:\/\//i.test(n);if(i&&!a)return n;if(!i&&a)return r;return n.length>r.length?n:r}function he(t){if(!t)return null;try{let e=new URL(t);if(!["http:","https:"].includes(e.protocol))return null;if(e.hash="",e.hostname=e.hostname.toLowerCase(),e.protocol==="https:"&&e.port==="443"||e.protocol==="http:"&&e.port==="80")e.port="";for(let i of[...e.searchParams.keys()]){let a=i.toLowerCase();if(Gn.includes(a)||a.startsWith("utm_"))e.searchParams.delete(i)}e.searchParams.sort();let r=e.pathname.replace(/\/{1,10}$/,"")||"/";e.pathname=r;let n=e.toString();return r==="/"?n.replace(/\/$/,""):n}catch{return null}}function qn(t){try{return new URL(t).hostname.toLowerCase().replace(/^www\./,"")}catch{return""}}function oe(t,e){return e.some((r)=>t===r||t.endsWith(`.${r}`))}function Jn(t,e="",r=""){let n=e.toLowerCase(),i=r.toLowerCase();if(t==="github.com"||t==="gitlab.com")return"repo";if(t==="arxiv.org"||t==="doi.org"||t==="semanticscholar.org"||t.endsWith(".semanticscholar.org")||i.includes("/paper/")||i.includes("/pdf/"))return"academic";if(oe(t,Mt))return"social";if(oe(t,Fn))return"community";if(oe(t,Hn))return"news";if(t.startsWith("docs.")||t.startsWith("developer.")||t.startsWith("developers.")||t.startsWith("api.")||n.includes("documentation")||n.includes("docs")||n.includes("reference")||i.includes("/docs/")||i.includes("/reference/")||i.includes("/api/"))return"official-docs";if(t.startsWith("blog.")||i.includes("/blog/"))return"maintainer-blog";return"website"}function Yn(t){switch(t){case"official-docs":return 5;case"repo":return 4;case"academic":return 4;case"maintainer-blog":return 3;case"website":return 2;case"community":return 1;case"news":return 0;case"social":return-6;default:return 0}}function Wn(t){let e=Object.values(t.perEngine||{}).map((r)=>r?.rank||99);return e.length?Math.min(...e):99}var Bn=["reddit.com","news.ycombinator.com","lobste.rs"];function ge(t){return t.smartScore*3+t.engineCount*5+Yn(t.sourceType)*2+Math.max(0,7-Wn(t))}function Qn(t){let e=t.toLowerCase(),r=[];if(e.includes("openai")||e.includes("gpt")||e.includes("chatgpt"))r.push("openai.com","platform.openai.com","help.openai.com");if(e.includes("anthropic")||e.includes("claude"))r.push("anthropic.com","docs.anthropic.com");if(e.includes("bun"))r.push("bun.sh","bun.com");if(e.includes("next.js")||e.includes("nextjs"))r.push("nextjs.org","vercel.com");if(e.includes("playwright"))r.push("playwright.dev");if(e.includes("supabase"))r.push("supabase.com","supabase.io");if(e.includes("prisma"))r.push("prisma.io");if(e.includes("tailwind"))r.push("tailwindcss.com");if(e.includes("vite"))r.push("vitejs.dev","vite.dev");if(e.includes("astro"))r.push("astro.build");if(e.includes("svelte"))r.push("svelte.dev");if(e.includes("solid"))r.push("solidjs.com");if(e.includes("vue")||e.includes("nuxt"))r.push("vuejs.org","nuxt.com");if(e.includes("react")||e.includes("react native"))r.push("react.dev","reactnative.dev");if(e.includes("angular"))r.push("angular.io","angular.dev");if(e.includes("node.js")||e.includes("nodejs"))r.push("nodejs.org","nodejs.dev","npmjs.com");if(/\bgo\b/.test(e)||e.includes("golang"))r.push("go.dev","golang.org","pkg.go.dev");if(e.includes("deno"))r.push("deno.land","deno.com");if(e.includes("fresh"))r.push("fresh.deno.dev");if(e.includes("typescript")||e.includes("ts"))r.push("typescriptlang.org");if(e.includes("python"))r.push("python.org","docs.python.org");if(e.includes("rust"))r.push("rust-lang.org","docs.rs","crates.io");if(e.includes("zig"))r.push("ziglang.org");if(e.includes("docker"))r.push("docker.com","docs.docker.com","hub.docker.com");if(e.includes("kubernetes")||e.includes("k8s"))r.push("kubernetes.io","k8s.io");if(e.includes("postgres")||e.includes("postgresql"))r.push("postgresql.org","neon.tech","supabase.com");if(e.includes("redis"))r.push("redis.io");if(e.includes("sqlite"))r.push("sqlite.org");if(e.includes("cloudflare"))r.push("developers.cloudflare.com","cloudflare.com");if(e.includes("vercel"))r.push("vercel.com","nextjs.org");if(e.includes("netlify"))r.push("netlify.com","docs.netlify.com");if(e.includes("stripe"))r.push("stripe.com","docs.stripe.com");if(e.includes("github"))r.push("github.com","docs.github.com");if(e.includes("gitlab"))r.push("gitlab.com","docs.gitlab.com");if(e.includes("aws"))r.push("aws.amazon.com","docs.aws.amazon.com");if(e.includes("azure"))r.push("azure.microsoft.com","learn.microsoft.com");if(e.includes("gcp")||e.includes("google cloud"))r.push("cloud.google.com","developers.google.com");if(e.includes("gemini")||e.includes("google ai"))r.push("ai.google.dev","developers.google.com");for(let n of Mt){let i=n.replace(/\.com$/,"");if(e.includes(i))r.push(n)}return[...new Set(r)]}function Lt(t,e){return t===e||t.endsWith(`.${e}`)}function Be(t,e=""){let r=new Map,n=Object.keys(t||{}).filter((c)=>!c.startsWith("_")),i=Qn(e);for(let c of n){let u=t[c];if(!u?.sources)continue;for(let d=0;d<u.sources.length;d++){let p=u.sources[d],f=he(p.url);if(!f||f.length<10)continue;let m=Ye(p.title||""),g=qn(f),h=Jn(g,m,f),y=0;if(i.some((E)=>Lt(g,E)))y+=10;if(h==="official-docs")y+=3;let v=f.toLowerCase();if(/\/docs\/|\/documentation\/|\.dev\/|\/api\/|\/reference\//.test(v))y+=2;let b=i.some((E)=>Lt(g,E));if(h==="social"&&!b)y-=20;if(i.length>0){if(oe(g,Bn))y-=3;else if(h==="community"&&!oe(g,["stackoverflow.com","stackexchange.com"]))y-=1}let w=r.get(f)||{id:"",canonicalUrl:f,displayUrl:p.url||f,domain:g,title:"",engines:[],engineCount:0,perEngine:{},sourceType:h,isOfficial:h==="official-docs",smartScore:0};if(w.title=We(w.title,m),w.displayUrl=w.displayUrl||p.url||f,w.sourceType=w.sourceType||h,w.isOfficial=w.isOfficial||h==="official-docs",w.smartScore=Math.max(w.smartScore,y),!w.engines.includes(c))w.engines.push(c);w.perEngine[c]={rank:d+1,title:We(w.perEngine[c]?.title||"",m)},r.set(f,w)}}let a=Array.from(r.values()).map((c)=>({...c,engineCount:c.engines.length})),o=a.filter((c)=>c.sourceType!=="social"),s=a.filter((c)=>c.sourceType==="social");return o.sort((c,u)=>{let d=ge(u)-ge(c);if(d!==0)return d;return c.domain.localeCompare(u.domain)}),s.sort((c,u)=>{let d=ge(u)-ge(c);if(d!==0)return d;return c.domain.localeCompare(u.domain)}),[...o,...s].slice(0,12).map((c,u)=>({...c,id:`S${u+1}`,title:c.title||c.domain||c.canonicalUrl}))}function Ut(t,e){let r=new Map(e.map((n)=>[n.id,n]));return t.map((n)=>{let i=r.get(n.id);if(!i)return n;let a=We(n.title,i.title||"");return{...n,title:a||n.title,fetch:{attempted:!0,ok:!i.error&&i.contentChars>100,status:i.status||null,finalUrl:i.finalUrl||i.url||n.canonicalUrl,contentType:i.contentType||"",lastModified:i.lastModified||"",publishedTime:i.publishedTime||"",byline:i.byline||"",siteName:i.siteName||"",lang:i.lang||"",title:i.title||"",snippet:i.snippet||"",contentChars:i.contentChars||0,source:i.source||"unknown",duration:i.duration||0,error:i.error||""}}})}import{createRequire as Vn}from"node:module";import{mkdirSync as Xn,writeFileSync as ei}from"node:fs";import{join as nr}from"node:path";import{createRequire as ni}from"node:module";import{createRequire as ii}from"node:module";import{spawn as ai}from"node:child_process";import{basename as oi}from"node:path";import{dirname as si,join as li}from"node:path";import{fileURLToPath as ci}from"node:url";import{existsSync as $e,mkdirSync as ui,readFileSync as or,writeFileSync as di}from"node:fs";import{homedir as fi}from"node:os";import{join as Gt}from"node:path";import{tmpdir as pi}from"node:os";import{existsSync as _e,mkdirSync as gi,readFileSync as sr,writeFileSync as hi}from"node:fs";import{homedir as mi}from"node:os";import{join as Ft}from"node:path";import{tmpdir as yi}from"node:os";import{existsSync as Ee,mkdirSync as wi,readFileSync as lr,writeFileSync as bi}from"node:fs";import{homedir as vi}from"node:os";import{join as Ht}from"node:path";import{tmpdir as $i}from"node:os";import{basename as eo}from"node:path";import{mkdirSync as Qe,writeFileSync as C}from"node:fs";import{join as _}from"node:path";import{fileURLToPath as ro}from"node:url";import{existsSync as Oe,mkdirSync as io,readFileSync as Or,writeFileSync as ao}from"node:fs";import{homedir as oo}from"node:os";import{join as Nr}from"node:path";import{tmpdir as so}from"node:os";import{spawn as mo}from"node:child_process";import{existsSync as Pr}from"node:fs";import{dirname as yo,join as ye}from"node:path";import{fileURLToPath as wo}from"node:url";import{join as Ao}from"node:path";import{fileURLToPath as Co}from"node:url";var Kn=Object.defineProperty,zn=(t)=>t;function Zn(t,e){this[t]=zn.bind(null,e)}var ut=(t,e)=>{for(var r in e)Kn(t,r,{get:e[r],enumerable:!0,configurable:!0,set:Zn.bind(e,r)})},dt=(t,e)=>()=>(t&&(e=t(t=0)),e),rc=Vn(import.meta.url),rr={};ut(rr,{writeSourcesToFiles:()=>ti});function ti(t,e=ir){return Xn(e,{recursive:!0}),t.map((r)=>{if(!r.content||r.content.length<10)return r;let n=String(r.id||"unknown").replace(/[^a-zA-Z0-9_-]/g,""),i=(r.canonicalUrl||r.url||"").replace(/^https?:\/\//,"").replace(/[^a-zA-Z0-9]/g,"-").slice(0,40),a=`${n}-${i}.md`,o=nr(e,a),s=`---
|
|
9
|
+
url: ${r.finalUrl||r.url}
|
|
10
|
+
title: ${r.title||""}
|
|
11
|
+
source: ${r.source||"unknown"}
|
|
12
|
+
status: ${r.status||""}
|
|
13
|
+
chars: ${r.contentChars||r.content.length}
|
|
14
|
+
---
|
|
15
|
+
|
|
16
|
+
`;ei(o,s+r.content,"utf8");let{content:l,...c}=r;return{...c,contentPath:o,contentChars:r.contentChars||l.length}})}var ir,ar=dt(()=>{ir=nr(process.cwd(),".dm","greedysearch-sources")}),ri={};ut(ri,{fetchTopSource:()=>ia,fetchSourceContent:()=>yr,fetchMultipleSources:()=>na});async function _i(){if(me)return me;let[{Readability:t},{JSDOM:e},{default:r}]=await Promise.all([import("@mozilla/readability"),import("jsdom"),import("turndown")]),n=new r({headingStyle:"atx",bulletListMarker:"-",codeBlockStyle:"fenced"});return n.addRule("removeDataUrls",{filter:(i)=>i.tagName==="IMG"&&i.getAttribute("src")?.startsWith("data:"),replacement:()=>""}),me={Readability:t,JSDOM:e,turndown:n},me}function Ei(t){if(!t||t.includes(":"))return null;let e=t.split(".");if(e.length<1||e.length>4)return null;if(!e.every((i)=>_r.test(i)))return null;let r=e.map((i)=>{if(/^0x/i.test(i))return parseInt(i,16);if(/^0[0-7]+$/.test(i))return parseInt(i,8);return parseInt(i,10)});if(r.some((i)=>!Number.isFinite(i)||i<0))return null;let n;if(r.length===1){if(r[0]>4294967295)return null;n=[r[0]>>>24&255,r[0]>>>16&255,r[0]>>>8&255,r[0]&255]}else if(r.length===2){if(r[0]>255||r[1]>16777215)return null;n=[r[0],r[1]>>>16&255,r[1]>>>8&255,r[1]&255]}else if(r.length===3){if(r[0]>255||r[1]>255||r[2]>65535)return null;n=[r[0],r[1],r[2]>>>8&255,r[2]&255]}else{if(r.some((i)=>i>255))return null;n=r}return n.join(".")}function Si(t){let e=t.match(vr);if(e)return`${e[1]}.${e[2]}.${e[3]}.${e[4]}`;let r=t.match($r);if(r){let n=parseInt(r[1],16),i=parseInt(r[2],16);if(n>65535||i>65535)return null;return[n>>>8&255,n&255,i>>>8&255,i&255].join(".")}return null}function qt(t){return ht.some((e)=>e.test(t))}function Ri(t={}){return{...gt,...t}}function ze(t){try{if(typeof t!=="string"||!t.trim())return{blocked:!0,reason:"URL must be a non-empty string"};let e=new URL(t);if(e.protocol!=="http:"&&e.protocol!=="https:")return{blocked:!0,reason:`Protocol not allowed: ${e.protocol}`};let r=e.hostname.toLowerCase();for(let i of ht)if(i.test(r))return{blocked:!0,reason:`Private/internal address: ${r}`};if(r.startsWith("[")&&r.endsWith("]")){let i=r.slice(1,-1),a=Si(i);if(a&&qt(a))return{blocked:!0,reason:`Private/internal address: ${r} (maps to ${a})`}}let n=Ei(r);if(n&&qt(n))return{blocked:!0,reason:`Private/internal address: ${r} (normalizes to ${n})`};return{blocked:!1}}catch(e){return{blocked:!0,reason:`Invalid URL: ${e.message}`}}}function Di(t){try{let e=new URL(t);if(!(e.hostname==="github.com"||e.hostname.endsWith(".github.com")))return t;let r=e.pathname.split("/").filter(Boolean);if(r.length<5)return t;let[n,i,a,o,...s]=r;if(a!=="blob")return t;let l=s.join("/");return`https://raw.githubusercontent.com/${n}/${i}/${o}/${l}`}catch{return t}}async function Ai(t,e={}){let r=ze(t);if(r.blocked)return{ok:!1,url:t,finalUrl:t,status:403,error:`Blocked: ${r.reason}`,needsBrowser:!1};let n=t;if(t=Di(t),t!==n)console.error(`[fetcher] Rewrote GitHub URL: ${n.slice(0,60)}... → raw.githubusercontent.com`);let{timeoutMs:i=15000,userAgent:a,signal:o}=e,s=new AbortController,l=setTimeout(()=>s.abort(),i);if(o)o.addEventListener("abort",()=>s.abort(),{once:!0});try{let c=await fetch(t,{method:"GET",headers:{...gt,"user-agent":a||br},redirect:"follow",signal:s.signal});clearTimeout(l);let u=c.headers.get("content-type")||"",d=c.url,p=c.headers.get("last-modified")||"",f=ze(d);if(f.blocked)return{ok:!1,url:t,finalUrl:d,status:c.status,error:`Blocked: ${f.reason}`,needsBrowser:!1};let m=!1;try{m=new URL(d).hostname.toLowerCase()==="raw.githubusercontent.com"}catch{}if(u.includes("text/plain")&&m){let b=await c.text();return{ok:!0,url:n,finalUrl:d,status:c.status,title:d.split("/").pop()||"GitHub File",byline:"",siteName:"GitHub",lang:"",publishedTime:p,lastModified:p,markdown:b,contentLength:b.length,excerpt:b.slice(0,300).replaceAll(/\n/g," "),needsBrowser:!1}}if(!u.includes("text/html")&&!u.includes("application/xhtml"))return{ok:!1,url:t,finalUrl:d,status:c.status,error:`Unsupported content type: ${u}`,needsBrowser:!1};let g=await c.text(),h=cr(c.status,g,d,t);if(h.blocked)return{ok:!1,url:t,finalUrl:d,status:c.status,error:`Blocked: ${h.reason}`,needsBrowser:!0};let y=await ur(g,d),v=dr(y);if(!v.ok)return{ok:!1,url:t,finalUrl:d,status:c.status,error:`Low quality content: ${v.reason}`,needsBrowser:!0};return{ok:!0,url:t,finalUrl:d,status:c.status,title:y.title,byline:y.byline,siteName:y.siteName,lang:y.lang,publishedTime:y.publishedTime||p,lastModified:p,markdown:y.markdown,excerpt:y.excerpt,contentLength:y.markdown.length,needsBrowser:!1}}catch(c){clearTimeout(l);let u=ki(c);return{ok:!1,url:t,finalUrl:t,status:0,error:c.message,needsBrowser:u}}}function cr(t,e,r,n){let i=e.match(/<title[^>]*>([^<]*)<\/title>/i)?.[1]?.toLowerCase()||"",a=e.slice(0,30000).toLowerCase(),o=`${i} ${a}`;if(t===403||t===429||t===503)return{blocked:!0,reason:`HTTP ${t}`};let s=[{pattern:/class=["'][^"']*captcha["']|<div[^>]*id=["']captcha/i,reason:"captcha"},{pattern:/g-recaptcha|data-sitekey|i['"]m not a robot/i,reason:"captcha"},{pattern:/checking your browser.{0,100}please wait|cf-browser-verification/i,reason:"cloudflare challenge"},{pattern:/just a moment.{0,50}security check|ddos protection by cloudflare/i,reason:"cloudflare challenge"},{pattern:/unusual traffic.{0,50}from your computer network/i,reason:"unusual traffic"},{pattern:/bot detected|automated.{0,20}request/i,reason:"bot detection"},{pattern:/enable\s+javascript\s+to\s+view|javascript\s+is\s+required.{0,50}enabled/i,reason:"requires javascript"},{pattern:/access denied|accessdenied/i,reason:"access denied"},{pattern:/protected by anubis|anubis uses a proof-of-work/i,reason:"anubis challenge"}];for(let c of s)if(c.pattern.test(o))return{blocked:!0,reason:c.reason};let l=Ci(n,r,e);if(l)return{blocked:!0,reason:l};return{blocked:!1}}function Ci(t,e,r){try{let n=new URL(t),i=new URL(e);if(n.hostname.toLowerCase()===i.hostname.toLowerCase())return;let a=i.hostname.toLowerCase();if(Er.some((s)=>a===s||a.endsWith(`.${s}`)))return`redirected to login (${i.hostname})`;if(Sr.some((s)=>a.startsWith(s)))return`redirected to login (${i.hostname})`;let o=r.slice(0,20000).toLowerCase();if(Rr.some((s)=>o.includes(s)))return`redirected to login page (${i.hostname})`}catch{}return}function ki(t){let e=t.message.toLowerCase();return e.includes("fetch failed")||e.includes("unable to verify")||e.includes("certificate")||e.includes("timeout")}function Jt(t){let e=['meta[property="article:published_time"]','meta[name="article:published_time"]','meta[property="og:published_time"]','meta[name="publication_date"]','meta[name="date"]','meta[itemprop="datePublished"]','time[itemprop="datePublished"]','meta[name="DC.date"]'];for(let r of e){let n=t.querySelector(r),i=n?.getAttribute("content")||n?.getAttribute("datetime")||"";if(i)return i}return""}async function ur(t,e){let{Readability:r,JSDOM:n,turndown:i}=await _i(),a=new n(t,{url:e});try{let o=a.window.document,s=new r(o).parse();if(s&&s.content){let c=i.turndown(s.content).replaceAll(/\n{3,}/g,`
|
|
17
|
+
|
|
18
|
+
`).trim(),u=s.publishedTime||Jt(o)||"";return{title:s.title||o.title||e,byline:s.byline||"",siteName:s.siteName||"",lang:s.lang||"",publishedTime:u,markdown:c,excerpt:c.slice(0,300).replaceAll(/\n/g," ")}}let l=o.body;if(l){let c=l.cloneNode(!0);c.querySelectorAll("script, style, nav, footer, header, aside").forEach((d)=>d.remove());let u=(c.textContent||"").replaceAll(/\s+/g," ").trim();return{title:o.title||e,byline:"",siteName:"",lang:"",publishedTime:Jt(o),markdown:u,excerpt:u.slice(0,300)}}return{title:e,byline:"",siteName:"",lang:"",publishedTime:"",markdown:"",excerpt:""}}finally{a.window.close()}}function dr(t){let e=t.markdown.trim().toLowerCase(),r=(t.title||"").toLowerCase();if(t.markdown.trim().length<100)return{ok:!1,reason:"content too short (< 100 chars)"};let n=e.toLowerCase(),i=[{check:()=>n.includes("loading")&&n.includes("please wait"),desc:"loading page"},{check:()=>n.includes("please ensure javascript is enabled"),desc:"requires javascript"},{check:()=>n.includes("enable javascript to view"),desc:"requires javascript"},{check:()=>n.includes("just a moment"),desc:"cloudflare challenge detected in content"},{check:()=>n.includes("verify you are human"),desc:"human verification"},{check:()=>n.includes("captcha required"),desc:"captcha in extracted content"},{check:()=>n.includes("access denied"),desc:"access denied in content"},{check:()=>/^\s{0,10}sign\s{1,5}in\s{0,10}$|^\s{0,10}log\s{1,5}in\s{0,10}$/im.test(e),desc:"login form only"}];for(let{check:a,desc:o}of i)if(a())return{ok:!1,reason:o};if(r.includes("just a moment")||r.includes("checking your browser"))return{ok:!1,reason:"cloudflare challenge page detected in title"};return{ok:!0}}function Ze(t){try{let e=new URL(t);if(!(e.hostname==="github.com"||e.hostname.endsWith(".github.com")))return null;let r=e.pathname.split("/").filter(Boolean);if(r.length<2)return null;let[n,i]=r;if(r.length===2)return{owner:n,repo:i,type:"root"};if(r.length>=4&&(r[2]==="blob"||r[2]==="tree")){let a=r[2],o=r[3],s=r.slice(4).join("/");return{owner:n,repo:i,type:a,ref:o,path:s}}return null}catch{return null}}async function P(t,e=1e4){let r=new AbortController,n=setTimeout(()=>r.abort(),e);try{let i=await fetch(`https://api.github.com${t}`,{headers:mt,signal:r.signal});if(clearTimeout(n),!i.ok)throw Error(`GitHub API ${i.status}: ${t}`);return await i.json()}catch(i){throw clearTimeout(n),i}}async function Oi(t,e){try{let r=await P(`/repos/${t}/${e}/readme`);if(r.content&&r.encoding==="base64")return Buffer.from(r.content,"base64").toString("utf8");return""}catch{return""}}async function Yt(t,e,r="HEAD",n="",i){try{let a;if(r==="HEAD")if(i)a=await P(`/repos/${t}/${e}/git/ref/heads/${i}`).catch(()=>null);else a=await Promise.any([P(`/repos/${t}/${e}/git/ref/heads/main`),P(`/repos/${t}/${e}/git/ref/heads/master`)]).catch(()=>null);else a=await P(`/repos/${t}/${e}/git/ref/heads/${r}`).catch(()=>P(`/repos/${t}/${e}/git/ref/heads/master`).catch(()=>null));if(!a?.object?.sha)return[];let o=(await P(`/repos/${t}/${e}/git/commits/${a.object.sha}`)).tree.sha,s=(await P(`/repos/${t}/${e}/git/trees/${o}`)).tree||[];if(n)s=s.filter((l)=>l.path.startsWith(n));return s.slice(0,50).map((l)=>({path:l.path,type:l.type==="tree"?"dir":"file",size:l.size}))}catch{return[]}}async function Ni(t,e,r,n,i=1e4,a){let o=async(l)=>{let c=new AbortController,u=setTimeout(()=>c.abort(),i);try{let d=await fetch(l,{headers:{"user-agent":mt["user-agent"]},signal:c.signal});if(clearTimeout(u),d.ok)return await d.text();throw Error("not ok")}catch{throw clearTimeout(u),Error("failed")}};if(!r||r==="HEAD"){if(a)try{return await o(`https://raw.githubusercontent.com/${t}/${e}/${a}/${n}`)}catch{return null}try{return await Promise.any([o(`https://raw.githubusercontent.com/${t}/${e}/main/${n}`),o(`https://raw.githubusercontent.com/${t}/${e}/master/${n}`)])}catch{return null}}let s=[`https://raw.githubusercontent.com/${t}/${e}/${r}/${n}`,`https://raw.githubusercontent.com/${t}/${e}/master/${n}`];for(let l of s)try{return await o(l)}catch{}return null}async function xi(t){let e=Ze(t);if(!e)return{ok:!1,error:"Not a valid GitHub URL"};let{owner:r,repo:n,type:i,ref:a,path:o}=e;try{if(i==="root"||i==="tree"&&!o){let s=await P(`/repos/${r}/${n}`),[l,c]=await Promise.allSettled([Oi(r,n),Yt(r,n,a||"HEAD","",s.default_branch)]),u=l.status==="fulfilled"?l.value:"",d=c.status==="fulfilled"?c.value:[],p=s?.description?`
|
|
19
|
+
|
|
20
|
+
> ${s.description}`:"",f=s?.stargazers_count==null?"":` ⭐ ${s.stargazers_count}`,m=s?.language?` · ${s.language}`:"",g=`# ${r}/${n}${f}${m}${p}
|
|
21
|
+
|
|
22
|
+
`;if(u)g+=u.slice(0,6000);else g+=`[No README found]
|
|
23
|
+
|
|
24
|
+
Files:
|
|
25
|
+
${d.map((h)=>` ${h.type==="dir"?"\uD83D\uDCC1":"\uD83D\uDCC4"} ${h.path}`).join(`
|
|
26
|
+
`)}`;return{ok:!0,title:`${r}/${n}`,content:g,tree:d.slice(0,30)}}if(i==="blob"&&o){let s;if(!a||a==="HEAD")try{s=(await P(`/repos/${r}/${n}`)).default_branch}catch{s=void 0}let l=await Ni(r,n,a,o,1e4,s);if(l===null)return{ok:!1,error:`File not found: ${o}`};return{ok:!0,title:`${r}/${n}: ${o}`,content:l}}if(i==="tree"&&o){let s;if(!a||a==="HEAD")try{s=(await P(`/repos/${r}/${n}`)).default_branch}catch{s=void 0}let l=await Yt(r,n,a||"HEAD",o,s),c=l.map((u)=>` ${u.type==="dir"?"\uD83D\uDCC1":"\uD83D\uDCC4"} ${u.path}`).join(`
|
|
27
|
+
`);return{ok:!0,title:`${r}/${n}/${o}`,content:`[Directory: ${o}]
|
|
28
|
+
|
|
29
|
+
Files:
|
|
30
|
+
${c}`,tree:l}}return{ok:!1,error:"Unsupported GitHub URL type"}}catch(s){return{ok:!1,error:s.message}}}function Pi(t){try{let e=new URL(t),r=e.hostname.toLowerCase();if(!(r==="reddit.com"||r.endsWith(".reddit.com")))return null;let n=e.pathname;if(n.match(/^\/(u|user)\/[^/]+\/?$/i))return{type:"user",cleanUrl:Wt(t)};if(n.match(/^\/r\/[^/]+\/comments\/[^/]+/i))return{type:"post",cleanUrl:Wt(t)};return null}catch{return null}}function Wt(t){try{let e=new URL(t);return`${e.protocol}//${e.hostname}${e.pathname}`}catch{return t}}async function Ii(t,e=8000){let r=Date.now();try{let n=t.replace(/\/+$/,"")+".json",i=new AbortController,a=setTimeout(()=>i.abort(),15000),o=await fetch(n,{headers:Dr,signal:i.signal});if(clearTimeout(a),!o.ok)throw Error(`Reddit API ${o.status}`);let s=await o.json();if(!Array.isArray(s)||s.length<1)throw Error("Invalid Reddit API response structure");let l=s[0],c=s[1],u=l?.data?.children?.[0]?.data;if(!u)throw Error("No post data in Reddit response");let d=ji(u,c,e);return{ok:!0,url:t,finalUrl:t,status:200,contentType:"text/markdown",lastModified:"",title:u.title||"Reddit Post",byline:`u/${u.author}`,siteName:`r/${u.subreddit}`,lang:"en",publishedTime:new Date(u.created_utc*1000).toISOString(),excerpt:u.selftext?.slice(0,300).replace(/\n/g," ")||"",markdown:d,contentLength:d.length,needsBrowser:!1,duration:Date.now()-r}}catch(n){return{ok:!1,url:t,finalUrl:t,status:0,error:`Reddit fetch failed: ${n.message}`,needsBrowser:!1,duration:Date.now()-r}}}function ji(t,e,r){let n="";if(n+=`# ${t.title}
|
|
31
|
+
|
|
32
|
+
`,n+=`**Subreddit:** r/${t.subreddit} | **Author:** u/${t.author} | **Score:** ${t.score}
|
|
33
|
+
|
|
34
|
+
`,t.selftext)n+=t.selftext,n+=`
|
|
35
|
+
|
|
36
|
+
`;else if(t.url)try{let i=new URL(t.url).hostname.toLowerCase();if(i!=="reddit.com"&&!i.endsWith(".reddit.com"))n+=`**Link:** ${t.url}
|
|
37
|
+
|
|
38
|
+
`}catch{n+=`**Link:** ${t.url}
|
|
39
|
+
|
|
40
|
+
`}if(e?.data?.children?.length>0){n+=`---
|
|
41
|
+
|
|
42
|
+
## Comments
|
|
43
|
+
|
|
44
|
+
`;let i=e.data.children.filter((a)=>a.kind==="t1").slice(0,10);for(let a of i)n+=fr(a.data,0),n+=`
|
|
45
|
+
`}if(n.length>r)n=n.slice(0,r).trim()+`
|
|
46
|
+
|
|
47
|
+
... (truncated)`;return n}function fr(t,e){if(!t||t.body==="[deleted]"||t.body==="[removed]")return"";let r="> ".repeat(e),n="";if(n+=`${r}**u/${t.author}** (${t.score} pts)
|
|
48
|
+
`,n+=`${r}${t.body.replaceAll(`
|
|
49
|
+
`,`
|
|
50
|
+
`+r)}
|
|
51
|
+
`,e<3&&t.replies?.data?.children){let i=t.replies.data.children.filter((a)=>a.kind==="t1");for(let a of i.slice(0,5))n+=`
|
|
52
|
+
`+fr(a.data,e+1)}return n}function re(t,e=8000){if(!t||t.length<=e)return t;let r=`
|
|
53
|
+
|
|
54
|
+
[...content trimmed...]
|
|
55
|
+
|
|
56
|
+
`,n=e-r.length,i=Math.floor(n*0.75),a=n-i,o=i;while(o>i-100&&t[o]!==`
|
|
57
|
+
`)o--;if(o<=i-100)o=i;let s=t.length-a;while(s<t.length-a+100&&t[s]!==`
|
|
58
|
+
`)s++;if(s>=t.length-a+100)s=t.length-a;let l=t.slice(0,o).trimEnd(),c=t.slice(s).trimStart();return`${l}${r}${c}`}function Ti(t=process.env,e=process.execPath){let r=t.GREEDY_SEARCH_NODE||t.NODE_BINARY||t.NODE;if(r?.trim())return r.trim();let n=oi(e||"").toLowerCase();if(n==="node"||n==="node.exe")return e;return"node"}function Li(t){if(!Array.isArray(t)||t.length===0)throw Error("cdp: args must be a non-empty array");if(t[0]==="test")return t.map((e,r)=>Bt(e,r));if(!Cr.has(t[0]))throw Error(`cdp: unknown subcommand '${t[0]}'`);return t.map((e,r)=>Bt(e,r))}function Bt(t,e){if(typeof t!=="string")throw Error(`cdp: argv[${e}] must be a string (got ${typeof t})`);if(t.includes("\x00"))throw Error(`cdp: argv[${e}] contains a null byte`);return t}function pr(t,e=30000){return Mi(t,null,e)}function Mi(t,e=null,r=30000){let n=Li(t);return new Promise((i,a)=>{let o=ai(Ti(),[Ar,...n],{stdio:[e==null?"ignore":"pipe","pipe","pipe"]});if(e!=null)o.stdin.write(e),o.stdin.end();let s="",l="";o.stdout.on("data",(u)=>s+=u),o.stderr.on("data",(u)=>l+=u);let c=setTimeout(()=>{o.kill(),a(Error(`cdp timeout: ${t[0]}`))},r);o.on("close",(u)=>{if(clearTimeout(c),u===0)i(s.trim());else a(Error(l.trim()||`cdp exit ${u}`))})})}async function Qt(t){await pr(["evalraw",t,"Page.addScriptToEvaluateOnNewDocument",JSON.stringify({source:`
|
|
59
|
+
(function() {
|
|
60
|
+
// ── Runtime.enable / CDP detection masking ──────────────
|
|
61
|
+
try { delete window.__REBROWSER_RUNTIME_ENABLE; } catch(_) {}
|
|
62
|
+
try { delete window.__REBROWSER_DEVTOOLS; } catch(_) {}
|
|
63
|
+
try { delete window.__nightmare; } catch(_) {}
|
|
64
|
+
try { delete window.__phantom; } catch(_) {}
|
|
65
|
+
try { delete window.callPhantom; } catch(_) {}
|
|
66
|
+
try { delete window._phantom; } catch(_) {}
|
|
67
|
+
try { delete window.Buffer; } catch(_) {}
|
|
68
|
+
|
|
69
|
+
// Real Chrome without automation should not expose navigator.webdriver at all.
|
|
70
|
+
// A literal false or an own-property getter returning undefined is itself a
|
|
71
|
+
// common stealth tell; remove both instance and prototype properties when the
|
|
72
|
+
// descriptor is configurable (as it is with --disable-blink-features).
|
|
73
|
+
try { delete navigator.webdriver; } catch(_) {}
|
|
74
|
+
try { delete Navigator.prototype.webdriver; } catch(_) {}
|
|
75
|
+
Object.defineProperty(navigator, 'vendor', { get: () => 'Google Inc.', configurable: true });
|
|
76
|
+
Object.defineProperty(navigator, 'platform', { get: () => 'Win32', configurable: true });
|
|
77
|
+
Object.defineProperty(navigator, 'maxTouchPoints', { get: () => 0, configurable: true });
|
|
78
|
+
Object.defineProperty(navigator, 'pdfViewerEnabled', { get: () => true, configurable: true });
|
|
79
|
+
Object.defineProperty(navigator, 'productSub', { get: () => '20030107', configurable: true });
|
|
80
|
+
Object.defineProperty(navigator, 'product', { get: () => 'Gecko', configurable: true });
|
|
81
|
+
var __greedyMimeTypes = null;
|
|
82
|
+
function __makeMimeTypes() {
|
|
83
|
+
var pdf = { type: 'application/pdf', suffixes: 'pdf', description: 'Portable Document Format', enabledPlugin: null };
|
|
84
|
+
var textPdf = { type: 'text/pdf', suffixes: 'pdf', description: 'Portable Document Format', enabledPlugin: null };
|
|
85
|
+
try { Object.setPrototypeOf(pdf, MimeType.prototype); } catch(_) {}
|
|
86
|
+
try { Object.setPrototypeOf(textPdf, MimeType.prototype); } catch(_) {}
|
|
87
|
+
var m = [pdf, textPdf];
|
|
88
|
+
try { Object.setPrototypeOf(m, MimeTypeArray.prototype); } catch(_) {}
|
|
89
|
+
m.item = function item(i) { return this[i] || null; };
|
|
90
|
+
m.namedItem = function namedItem(name) { return Array.prototype.find.call(this, function(x) { return x && x.type === name; }) || null; };
|
|
91
|
+
return m;
|
|
92
|
+
}
|
|
93
|
+
Object.defineProperty(navigator, 'plugins', {
|
|
94
|
+
get: () => {
|
|
95
|
+
__greedyMimeTypes = __greedyMimeTypes || __makeMimeTypes();
|
|
96
|
+
var plugin0 = { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' };
|
|
97
|
+
var plugin1 = { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' };
|
|
98
|
+
var plugin2 = { name: 'Native Client', filename: 'internal-nacl-plugin', description: '' };
|
|
99
|
+
try { Object.setPrototypeOf(plugin0, Plugin.prototype); } catch(_) {}
|
|
100
|
+
try { Object.setPrototypeOf(plugin1, Plugin.prototype); } catch(_) {}
|
|
101
|
+
try { Object.setPrototypeOf(plugin2, Plugin.prototype); } catch(_) {}
|
|
102
|
+
var p = [plugin0, plugin1, plugin2];
|
|
103
|
+
p.item = function item(i) { return this[i] || null; };
|
|
104
|
+
p.namedItem = function namedItem(name) { return Array.prototype.find.call(this, function(x) { return x && x.name === name; }) || null; };
|
|
105
|
+
p.refresh = function refresh() {};
|
|
106
|
+
try { Object.setPrototypeOf(p, PluginArray.prototype); } catch(_) {}
|
|
107
|
+
try {
|
|
108
|
+
__greedyMimeTypes[0].enabledPlugin = p[0];
|
|
109
|
+
__greedyMimeTypes[1].enabledPlugin = p[0];
|
|
110
|
+
} catch(_) {}
|
|
111
|
+
return p;
|
|
112
|
+
},
|
|
113
|
+
configurable: true,
|
|
114
|
+
});
|
|
115
|
+
Object.defineProperty(navigator, 'mimeTypes', {
|
|
116
|
+
get: () => {
|
|
117
|
+
__greedyMimeTypes = __greedyMimeTypes || __makeMimeTypes();
|
|
118
|
+
return __greedyMimeTypes;
|
|
119
|
+
},
|
|
120
|
+
configurable: true,
|
|
121
|
+
});
|
|
122
|
+
Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'], configurable: true });
|
|
123
|
+
try {
|
|
124
|
+
Object.defineProperty(navigator, 'connection', { get: () => ({ effectiveType: '4g', rtt: 50, downlink: 10, downlinkMax: Infinity, saveData: false }), configurable: true });
|
|
125
|
+
} catch(_) {}
|
|
126
|
+
if (!navigator.mediaDevices) {
|
|
127
|
+
Object.defineProperty(navigator, 'mediaDevices', {
|
|
128
|
+
get: () => ({
|
|
129
|
+
enumerateDevices: () => Promise.resolve([
|
|
130
|
+
{ deviceId: 'default', kind: 'audioinput', label: '', groupId: 'default' },
|
|
131
|
+
{ deviceId: 'default', kind: 'audiooutput', label: '', groupId: 'default' },
|
|
132
|
+
{ deviceId: '', kind: 'videoinput', label: '', groupId: '' },
|
|
133
|
+
]),
|
|
134
|
+
getUserMedia: () => Promise.reject(new DOMException('NotAllowedError')),
|
|
135
|
+
getDisplayMedia: () => Promise.reject(new DOMException('NotAllowedError')),
|
|
136
|
+
}),
|
|
137
|
+
configurable: true,
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
// ── Missing platform APIs (headless often lacks these) ─
|
|
141
|
+
try {
|
|
142
|
+
if (!navigator.share) {
|
|
143
|
+
navigator.share = function() { return Promise.reject(new Error('NotAllowedError')); };
|
|
144
|
+
}
|
|
145
|
+
} catch(_) {}
|
|
146
|
+
try {
|
|
147
|
+
if (!navigator.contentIndex) {
|
|
148
|
+
Object.defineProperty(navigator, 'contentIndex', { get: () => ({ add: function() {}, delete: function() {}, getAll: function() { return Promise.resolve([]); } }), configurable: true });
|
|
149
|
+
}
|
|
150
|
+
} catch(_) {}
|
|
151
|
+
|
|
152
|
+
if (!window.chrome) {
|
|
153
|
+
window.chrome = {
|
|
154
|
+
app: { isInstalled: false, InstallState: {}, RunningState: {} },
|
|
155
|
+
runtime: {
|
|
156
|
+
OnInstalledReason: {}, OnRestartRequiredReason: {}, PlatformArch: {}, PlatformNaclArch: {}, PlatformOs: {}, RequestUpdateCheckStatus: {},
|
|
157
|
+
connect: () => ({}), sendMessage: () => {}, onMessage: { addListener: () => {} }
|
|
158
|
+
},
|
|
159
|
+
loadTimes: function() { return { requestTime: 0, startLoadTime: Date.now() - 5000, commitLoadTime: Date.now() - 3000, finishDocumentLoadTime: Date.now() - 2000, finishLoadTime: Date.now() - 1000, firstPaintTime: Date.now() - 800, navigationType: 'Other', wasFetchedViaSpdy: true, wasNpnNegotiated: true, npnNegotiatedProtocol: 'h2', wasAlternateProtocolAvailable: false, connectionInfo: 'http/2' }; },
|
|
160
|
+
csi: function() { var t = Date.now(); return { onloadT: t - 2000, startE: t - 5000, pageT: 'back', tran: 2 }; },
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
var __greedyNativeFns = [];
|
|
164
|
+
function __markNative(fn) { try { __greedyNativeFns.push(fn); } catch(_) {} return fn; }
|
|
165
|
+
|
|
166
|
+
var origQuery = navigator.permissions?.query;
|
|
167
|
+
if (origQuery) {
|
|
168
|
+
navigator.permissions.query = __markNative(function query(params) {
|
|
169
|
+
if (params && params.name === 'notifications') return Promise.resolve({ state: Notification.permission || 'default', onchange: null });
|
|
170
|
+
return origQuery.apply(this, arguments);
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
try {
|
|
174
|
+
var getParam = WebGLRenderingContext.prototype.getParameter;
|
|
175
|
+
WebGLRenderingContext.prototype.getParameter = __markNative(function getParameter(p) {
|
|
176
|
+
if (p === 37445) return 'Intel Inc.';
|
|
177
|
+
if (p === 37446) return 'Intel Iris OpenGL Engine';
|
|
178
|
+
return getParam.call(this, p);
|
|
179
|
+
});
|
|
180
|
+
} catch(_) {}
|
|
181
|
+
// ── WebGL readPixels noise ──────────────────────────
|
|
182
|
+
// CreepJS and other fingerprinters draw content with WebGL and read back the
|
|
183
|
+
// rendered pixels. Adding subtle noise breaks rendering-based fingerprinting.
|
|
184
|
+
try {
|
|
185
|
+
var origReadPixels = WebGLRenderingContext.prototype.readPixels;
|
|
186
|
+
WebGLRenderingContext.prototype.readPixels = __markNative(function readPixels(x, y, width, height, format, type, pixels) {
|
|
187
|
+
var result = origReadPixels.call(this, x, y, width, height, format, type, pixels);
|
|
188
|
+
if (pixels && pixels.length > 0) {
|
|
189
|
+
pixels[0] ^= 1;
|
|
190
|
+
}
|
|
191
|
+
return result;
|
|
192
|
+
});
|
|
193
|
+
} catch(_) {}
|
|
194
|
+
Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8, configurable: true });
|
|
195
|
+
Object.defineProperty(navigator, 'deviceMemory', { get: () => 8, configurable: true });
|
|
196
|
+
|
|
197
|
+
// ── Canvas fingerprint noise ─────────────────────────
|
|
198
|
+
// Headless rendering engines produce slightly different canvas output
|
|
199
|
+
// than headed Chrome. Subtle noise breaks hash-based fingerprinting.
|
|
200
|
+
try {
|
|
201
|
+
var __canvasNoise = ((Date.now() & 0xFF) | 1);
|
|
202
|
+
var origFill = CanvasRenderingContext2D.prototype.fillText;
|
|
203
|
+
CanvasRenderingContext2D.prototype.fillText = __markNative(function fillText() {
|
|
204
|
+
this.globalAlpha = 0.9995;
|
|
205
|
+
return origFill.apply(this, arguments);
|
|
206
|
+
});
|
|
207
|
+
} catch(_) {}
|
|
208
|
+
try {
|
|
209
|
+
var origStroke = CanvasRenderingContext2D.prototype.strokeText;
|
|
210
|
+
CanvasRenderingContext2D.prototype.strokeText = __markNative(function strokeText() {
|
|
211
|
+
this.globalAlpha = 0.9995;
|
|
212
|
+
return origStroke.apply(this, arguments);
|
|
213
|
+
});
|
|
214
|
+
} catch(_) {}
|
|
215
|
+
try {
|
|
216
|
+
var origToDataURL = HTMLCanvasElement.prototype.toDataURL;
|
|
217
|
+
HTMLCanvasElement.prototype.toDataURL = __markNative(function toDataURL() {
|
|
218
|
+
var ctx = this.getContext('2d');
|
|
219
|
+
if (ctx) {
|
|
220
|
+
// Spread noise across canvas to break hash-based fingerprinting.
|
|
221
|
+
// Uses a deterministic pattern so it's consistent per page load
|
|
222
|
+
// but varies between sessions.
|
|
223
|
+
var w = this.width, h = this.height;
|
|
224
|
+
if (w > 0 && h > 0) {
|
|
225
|
+
var imgData = ctx.getImageData(0, 0, Math.min(w, 4), Math.min(h, 4));
|
|
226
|
+
if (imgData && imgData.data) {
|
|
227
|
+
for (var __i = 0; __i < imgData.data.length; __i += 4) {
|
|
228
|
+
imgData.data[__i] ^= (__canvasNoise + __i) & 0xFF;
|
|
229
|
+
}
|
|
230
|
+
ctx.putImageData(imgData, 0, 0);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
return origToDataURL.apply(this, arguments);
|
|
235
|
+
});
|
|
236
|
+
} catch(_) {}
|
|
237
|
+
|
|
238
|
+
// ── AudioContext fingerprint noise ────────────────────
|
|
239
|
+
// Headless Chrome's AudioContext produces slightly different output.
|
|
240
|
+
// Subtle noise breaks audio-based fingerprinting.
|
|
241
|
+
try {
|
|
242
|
+
var __audioSeed = ((Date.now() & 0x1F) | 1);
|
|
243
|
+
var origGetChannelData = AudioBuffer.prototype.getChannelData;
|
|
244
|
+
AudioBuffer.prototype.getChannelData = __markNative(function getChannelData(channel) {
|
|
245
|
+
var data = origGetChannelData.call(this, channel);
|
|
246
|
+
for (var __i = 0; __i < data.length; __i += 64) {
|
|
247
|
+
data[__i] *= 0.99999;
|
|
248
|
+
}
|
|
249
|
+
return data;
|
|
250
|
+
});
|
|
251
|
+
} catch(_) {}
|
|
252
|
+
|
|
253
|
+
// ── window outer dimensions ──────────────────────────
|
|
254
|
+
// outerWidth/Height = 0 in headless — a well-known bot signal.
|
|
255
|
+
// Mirror innerWidth/Height (set by --window-size flag) so the ratio is sane.
|
|
256
|
+
try {
|
|
257
|
+
if (!window.outerWidth) Object.defineProperty(window, 'outerWidth', { get: () => window.innerWidth || 1920, configurable: true });
|
|
258
|
+
if (!window.outerHeight) Object.defineProperty(window, 'outerHeight', { get: () => window.innerHeight || 1080, configurable: true });
|
|
259
|
+
} catch(_) {}
|
|
260
|
+
|
|
261
|
+
// ── screen properties ─────────────────────────────────
|
|
262
|
+
// Headless Chrome often reports an 800x600 screen even when the viewport is
|
|
263
|
+
// 1920x1080. Keep screen metrics internally consistent with our launch flags.
|
|
264
|
+
try {
|
|
265
|
+
Object.defineProperty(screen, 'width', { get: () => 1920, configurable: true });
|
|
266
|
+
Object.defineProperty(screen, 'height', { get: () => 1080, configurable: true });
|
|
267
|
+
Object.defineProperty(screen, 'availWidth', { get: () => 1920, configurable: true });
|
|
268
|
+
Object.defineProperty(screen, 'availHeight', { get: () => 1040, configurable: true });
|
|
269
|
+
Object.defineProperty(screen, 'colorDepth', { get: () => 24, configurable: true });
|
|
270
|
+
Object.defineProperty(screen, 'pixelDepth', { get: () => 24, configurable: true });
|
|
271
|
+
} catch(_) {}
|
|
272
|
+
|
|
273
|
+
// ── navigator.userAgentData (UA Client Hints) ─────────
|
|
274
|
+
// Derive version from the UA string already set by --user-agent flag so the
|
|
275
|
+
// two APIs are always consistent. Removes any "HeadlessChrome" brand entry.
|
|
276
|
+
try {
|
|
277
|
+
var _uaMajor = (navigator.userAgent.match(new RegExp('Chrome/([0-9]+)')) || [])[1] || '136';
|
|
278
|
+
var _uaFull = (navigator.userAgent.match(new RegExp('Chrome/([0-9.]+)')) || [])[1] || (_uaMajor + '.0.0.0');
|
|
279
|
+
var _brands = [
|
|
280
|
+
{ brand: 'Not)A;Brand', version: '99' },
|
|
281
|
+
{ brand: 'Google Chrome', version: _uaMajor },
|
|
282
|
+
{ brand: 'Chromium', version: _uaMajor },
|
|
283
|
+
];
|
|
284
|
+
Object.defineProperty(navigator, 'userAgentData', {
|
|
285
|
+
get: function() {
|
|
286
|
+
return {
|
|
287
|
+
brands: _brands, mobile: false, platform: 'Windows',
|
|
288
|
+
getHighEntropyValues: function() {
|
|
289
|
+
return Promise.resolve({
|
|
290
|
+
architecture: 'x86', bitness: '64',
|
|
291
|
+
brands: _brands,
|
|
292
|
+
fullVersionList: [
|
|
293
|
+
{ brand: 'Not)A;Brand', version: '99.0.0.0' },
|
|
294
|
+
{ brand: 'Google Chrome', version: _uaFull },
|
|
295
|
+
{ brand: 'Chromium', version: _uaFull },
|
|
296
|
+
],
|
|
297
|
+
mobile: false, model: '', platform: 'Windows',
|
|
298
|
+
platformVersion: '15.0.0', uaFullVersion: _uaFull, wow64: false,
|
|
299
|
+
});
|
|
300
|
+
},
|
|
301
|
+
toJSON: function() { return { brands: _brands, mobile: false, platform: 'Windows' }; },
|
|
302
|
+
};
|
|
303
|
+
},
|
|
304
|
+
configurable: true,
|
|
305
|
+
});
|
|
306
|
+
} catch(_) {}
|
|
307
|
+
|
|
308
|
+
// ── CDP Runtime serialization guard ──────────────────
|
|
309
|
+
// Sites detect CDP by putting a getter on Error.prototype.stack
|
|
310
|
+
// and checking if console.log triggers it (only happens when
|
|
311
|
+
// Runtime domain is enabled). We monkey-patch console methods to
|
|
312
|
+
// strip custom getters from arguments before they reach CDP.
|
|
313
|
+
try {
|
|
314
|
+
var _origLog = console.log, _origError = console.error,
|
|
315
|
+
_origWarn = console.warn, _origDebug = console.debug,
|
|
316
|
+
_origInfo = console.info;
|
|
317
|
+
var _safeArg = function(a) {
|
|
318
|
+
if (a instanceof Error) {
|
|
319
|
+
try { return new Error(a.message); } catch(_) { return a; }
|
|
320
|
+
}
|
|
321
|
+
return a;
|
|
322
|
+
};
|
|
323
|
+
console.log = __markNative(function log() { return _origLog.apply(console, Array.prototype.map.call(arguments, _safeArg)); });
|
|
324
|
+
console.error = __markNative(function error() { return _origError.apply(console, Array.prototype.map.call(arguments, _safeArg)); });
|
|
325
|
+
console.warn = __markNative(function warn() { return _origWarn.apply(console, Array.prototype.map.call(arguments, _safeArg)); });
|
|
326
|
+
console.debug = __markNative(function debug() { return _origDebug.apply(console, Array.prototype.map.call(arguments, _safeArg)); });
|
|
327
|
+
console.info = __markNative(function info() { return _origInfo.apply(console, Array.prototype.map.call(arguments, _safeArg)); });
|
|
328
|
+
} catch(_) {}
|
|
329
|
+
|
|
330
|
+
// ── Native function masking ──────────────────────────
|
|
331
|
+
// Patched APIs should not stringify as user-defined stealth code.
|
|
332
|
+
try {
|
|
333
|
+
var __nativeToString = Function.prototype.toString;
|
|
334
|
+
Function.prototype.toString = function toString() {
|
|
335
|
+
if (__greedyNativeFns.indexOf(this) !== -1) {
|
|
336
|
+
var name = this.name || '';
|
|
337
|
+
return 'function ' + name + '() { [native code] }';
|
|
338
|
+
}
|
|
339
|
+
return __nativeToString.call(this);
|
|
340
|
+
};
|
|
341
|
+
} catch(_) {}
|
|
342
|
+
})();
|
|
343
|
+
`})])}function Ui(t,e=9222){let r=Number.parseInt(String(t??""),10);return Number.isInteger(r)&&r>1024&&r<65535?r:e}function Gi(){try{if($e(L)){let t=or(L,"utf8"),e=JSON.parse(t);if(Array.isArray(e.engines)&&e.engines.length>0&&e.engines.every((r)=>typeof r==="string")){let r=e.engines.filter((i)=>we[i]),n=e.engines.filter((i)=>!we[i]);if(n.length>0)process.stderr.write(`[greedysearch] Warning: ignoring unknown engine(s) in ${L}: ${n.join(", ")}
|
|
344
|
+
[greedysearch] Available engines: ${Object.keys(we).join(", ")}
|
|
345
|
+
`);if(r.length>0)return r;process.stderr.write(`[greedysearch] Warning: no valid engines in ${L}, falling back to defaults: ${Re.join(", ")}
|
|
346
|
+
`)}}}catch{}return Re}function Fi(){try{if(!$e(Se))ui(Se,{recursive:!0});if(!$e(L))di(L,JSON.stringify({engines:Re,synthesizer:Xe},null,2)+`
|
|
347
|
+
`,"utf8")}catch{}}function Hi(){try{if($e(L)){let t=or(L,"utf8"),e=JSON.parse(t);if(typeof e.synthesizer==="string"){let r=e.synthesizer.toLowerCase();if(et.includes(r))return r;process.stderr.write(`[greedysearch] Warning: unknown synthesizer "${e.synthesizer}" in ${L}
|
|
348
|
+
[greedysearch] Available synthesizers: ${et.join(", ")}
|
|
349
|
+
[greedysearch] Falling back to default: ${Xe}
|
|
350
|
+
`)}}}catch{}return Xe}function qi(t,e=9222){let r=Number.parseInt(String(t??""),10);return Number.isInteger(r)&&r>1024&&r<65535?r:e}function Ji(){try{if(_e(M)){let t=sr(M,"utf8"),e=JSON.parse(t);if(Array.isArray(e.engines)&&e.engines.length>0&&e.engines.every((r)=>typeof r==="string")){let r=e.engines.filter((i)=>be[i]),n=e.engines.filter((i)=>!be[i]);if(n.length>0)process.stderr.write(`[greedysearch] Warning: ignoring unknown engine(s) in ${M}: ${n.join(", ")}
|
|
351
|
+
[greedysearch] Available engines: ${Object.keys(be).join(", ")}
|
|
352
|
+
`);if(r.length>0)return r;process.stderr.write(`[greedysearch] Warning: no valid engines in ${M}, falling back to defaults: ${Ae.join(", ")}
|
|
353
|
+
`)}}}catch{}return Ae}function Yi(){try{if(!_e(De))gi(De,{recursive:!0});if(!_e(M))hi(M,JSON.stringify({engines:Ae,synthesizer:tt},null,2)+`
|
|
354
|
+
`,"utf8")}catch{}}function Wi(){try{if(_e(M)){let t=sr(M,"utf8"),e=JSON.parse(t);if(typeof e.synthesizer==="string"){let r=e.synthesizer.toLowerCase();if(rt.includes(r))return r;process.stderr.write(`[greedysearch] Warning: unknown synthesizer "${e.synthesizer}" in ${M}
|
|
355
|
+
[greedysearch] Available synthesizers: ${rt.join(", ")}
|
|
356
|
+
[greedysearch] Falling back to default: ${tt}
|
|
357
|
+
`)}}}catch{}return tt}async function gr(){let t=(await R(["list"])).split(`
|
|
358
|
+
`)[0];if(!t)throw Error("No Chrome tabs found");return t.slice(0,8)}async function ft(t="about:blank"){let e=await gr(),r=new URL(t).hostname;if(r==="copilot.microsoft.com"||r==="www.perplexity.ai"||r==="perplexity.ai"||r.endsWith(".perplexity.ai")){let a=await R(["evalraw",e,"Target.createTarget",JSON.stringify({url:"about:blank"})]),{targetId:o}=JSON.parse(a),s=o.slice(0,8);if(await R(["list"]).catch(()=>null),r==="copilot.microsoft.com")await Qt(s);else Qt(s).catch(()=>{});return await R(["list"]).catch(()=>null),o}let n=await R(["evalraw",e,"Target.createTarget",JSON.stringify({url:t})]),{targetId:i}=JSON.parse(n);return await R(["list"]).catch(()=>null),i}async function pt(t){try{let e=await gr();await R(["evalraw",e,"Target.closeTarget",JSON.stringify({targetId:t})])}catch{}}function Bi(t,e=9222){let r=Number.parseInt(String(t??""),10);return Number.isInteger(r)&&r>1024&&r<65535?r:e}function Qi(){try{if(Ee(U)){let t=lr(U,"utf8"),e=JSON.parse(t);if(Array.isArray(e.engines)&&e.engines.length>0&&e.engines.every((r)=>typeof r==="string")){let r=e.engines.filter((i)=>ve[i]),n=e.engines.filter((i)=>!ve[i]);if(n.length>0)process.stderr.write(`[greedysearch] Warning: ignoring unknown engine(s) in ${U}: ${n.join(", ")}
|
|
359
|
+
[greedysearch] Available engines: ${Object.keys(ve).join(", ")}
|
|
360
|
+
`);if(r.length>0)return r;process.stderr.write(`[greedysearch] Warning: no valid engines in ${U}, falling back to defaults: ${ke.join(", ")}
|
|
361
|
+
`)}}}catch{}return ke}function Vi(){try{if(!Ee(Ce))wi(Ce,{recursive:!0});if(!Ee(U))bi(U,JSON.stringify({engines:ke,synthesizer:nt},null,2)+`
|
|
362
|
+
`,"utf8")}catch{}}function Ki(){try{if(Ee(U)){let t=lr(U,"utf8"),e=JSON.parse(t);if(typeof e.synthesizer==="string"){let r=e.synthesizer.toLowerCase();if(it.includes(r))return r;process.stderr.write(`[greedysearch] Warning: unknown synthesizer "${e.synthesizer}" in ${U}
|
|
363
|
+
[greedysearch] Available synthesizers: ${it.join(", ")}
|
|
364
|
+
[greedysearch] Falling back to default: ${nt}
|
|
365
|
+
`)}}}catch{}return nt}function zi(){if(typeof globalThis.DOMMatrix>"u")globalThis.DOMMatrix=class{constructor(t=void 0){}multiplySelf(){return this}preMultiplySelf(){return this}translateSelf(){return this}scaleSelf(){return this}rotateSelf(){return this}};if(typeof globalThis.ImageData>"u")globalThis.ImageData=class{constructor(t=void 0,e=0,r=0){this.data=t,this.width=e,this.height=r}};if(typeof globalThis.Path2D>"u")globalThis.Path2D=class{constructor(t=void 0){}}}async function Zi(){zi();let t=await import("pdf-parse"),e=t.PDFParse??t.default;if(!e)throw Error("pdf-parse did not export PDFParse");return e}async function Xi(t,e){try{let r=new(await Zi())({data:new Uint8Array(t)});await r.load();let n=await r.getText(),i=n.text?.trim();if(!i)return null;return{title:new URL(e).pathname.split("/").pop()||"Document.pdf",content:`## PDF Content (${n.total} pages)
|
|
366
|
+
|
|
367
|
+
${i}`,pages:n.total}}catch(r){return{error:r.message||String(r)}}}function hr(t="",e=240){let r=String(t).replaceAll(/\s+/g," ").trim();if(r.length<=e)return r;let n=r.slice(0,e),i=n.lastIndexOf(" ");return i>0?`${n.slice(0,i)}...`:`${n}...`}async function ea(t,e,r=8000){let n=Date.now();try{let i=(await R(["evalraw",t,"Page.getFrameTree","{}"]).then((p)=>JSON.parse(p)).catch(()=>null))?.frameTree?.frame?.id||void 0,a=await R(["evalraw",t,"Network.loadNetworkResource",JSON.stringify({frameId:i,url:e,options:{disableCache:!0,includeCredentials:!1}})],20000),o=JSON.parse(a).resource;if(!o?.success||!o.httpStatusCode)return{url:e,error:o?.netErrorName||o?.netError||"loadNetworkResource failed",source:"chrome",duration:Date.now()-n,needsFallback:!0};let s="";if(o.stream)try{let p=await R(["evalraw",t,"IO.read",JSON.stringify({handle:o.stream})],1e4);s=JSON.parse(p).data||"",await R(["evalraw",t,"IO.close",JSON.stringify({handle:o.stream})]).catch(()=>{})}catch{}if(!s||s.length<100)return{url:e,error:"Empty response body from Network.loadNetworkResource",source:"chrome",duration:Date.now()-n,needsFallback:!0};let l=cr(o.httpStatusCode,s,e,e);if(l.blocked)return{url:e,status:o.httpStatusCode,error:`Blocked: ${l.reason}`,source:"chrome",duration:Date.now()-n,needsBrowser:!0};let c=await ur(s,e),u=dr(c);if(!u.ok)return{url:e,status:o.httpStatusCode,error:`Low quality: ${u.reason}`,source:"chrome",duration:Date.now()-n,needsBrowser:!0};let d=re(c.markdown,r);return{url:e,finalUrl:e,status:o.httpStatusCode,contentType:"text/markdown",lastModified:"",publishedTime:c.publishedTime||"",byline:c.byline||"",siteName:c.siteName||"",lang:c.lang||"",title:c.title||e,snippet:c.excerpt,content:d,contentChars:d.length,source:"chrome",duration:Date.now()-n}}catch(i){return{url:e,error:i.message,source:"chrome",duration:Date.now()-n,needsFallback:!0}}}function mr(t){try{return new URL(t).pathname.toLowerCase().endsWith(".pdf")}catch{return!1}}async function ta(t,e=8000){let r=ze(t);if(r.blocked)return{url:t,finalUrl:t,status:403,error:`Blocked: ${r.reason}`,source:"pdf-http"};let n=new AbortController,i=setTimeout(()=>n.abort(),20000),a=Date.now();try{let o=await fetch(t,{method:"GET",redirect:"follow",signal:n.signal,headers:Ri({accept:"application/pdf,application/octet-stream;q=0.9,*/*;q=0.5"})});clearTimeout(i);let s=o.headers.get("content-type")||"",l=o.url||t,c=Number.parseInt(o.headers.get("content-length")||"0",10);if(o.status>=400)return{url:t,finalUrl:l,status:o.status,error:`HTTP ${o.status}`,source:"pdf-http",duration:Date.now()-a};if(!s.toLowerCase().includes("application/pdf")&&!mr(l))return null;if(c>31457280)return{url:t,finalUrl:l,status:o.status,error:`PDF too large: ${c} bytes`,source:"pdf-http",duration:Date.now()-a};let u=Buffer.from(await o.arrayBuffer()),d=await Xi(u,l);if(!d||d.error)return{url:t,finalUrl:l,status:o.status,error:d?.error||"PDF text extraction failed",source:"pdf-http",duration:Date.now()-a};let p=re(d.content,e);return{url:t,finalUrl:l,status:o.status,contentType:"application/pdf",lastModified:o.headers.get("last-modified")||"",title:d.title,snippet:hr(p,320),content:p,contentChars:p.length,pages:d.pages,source:"pdf-http",duration:Date.now()-a}}catch(o){return clearTimeout(i),{url:t,finalUrl:t,error:o.message||String(o),source:"pdf-http",duration:Date.now()-a}}}async function yr(t,e=8000){let r=Date.now();if(mr(t)){let i=await ta(t,e);if(i?.content||i?.status===403)return i}if(Ze(t)){let i=Ze(t);if(i&&(i.type==="root"||i.type==="tree"||i.type==="blob"&&!i.path?.includes("."))){let a=await xi(t);if(a.ok){let o=re(a.content,e);return{url:t,finalUrl:t,status:200,contentType:"text/markdown",lastModified:"",title:a.title,snippet:o.slice(0,320),content:o,contentChars:o.length,source:"github-api",...a.tree&&{tree:a.tree},duration:Date.now()-r}}process.stderr.write(`[greedysearch] GitHub API fetch failed, trying HTTP: ${a.error}
|
|
368
|
+
`)}}if(Pi(t)?.type==="post"){process.stderr.write(`[greedysearch] Using Reddit JSON API for: ${t.slice(0,60)}...
|
|
369
|
+
`);let i=await Ii(t,e);if(i.ok){let a=re(i.markdown,e);return{url:t,finalUrl:i.finalUrl,status:i.status,contentType:"text/markdown",lastModified:i.lastModified||"",publishedTime:i.publishedTime||"",byline:i.byline||"",siteName:i.siteName||"",lang:i.lang||"",title:i.title,snippet:i.excerpt,content:a,contentChars:a.length,source:"reddit-api",duration:Date.now()-r}}process.stderr.write(`[greedysearch] Reddit API fetch failed, falling back to HTTP: ${i.error}
|
|
370
|
+
`)}let n=await Ai(t,{timeoutMs:1e4});if(n.ok){let i=re(n.markdown,e);return{url:t,finalUrl:n.finalUrl,status:n.status,contentType:"text/markdown",lastModified:n.lastModified||"",publishedTime:n.publishedTime||"",byline:n.byline||"",siteName:n.siteName||"",lang:n.lang||"",title:n.title,snippet:n.excerpt,content:i,contentChars:i.length,source:"http",duration:Date.now()-r}}if(n.needsBrowser)try{let i=await ft();try{let a=await ea(i,t,e);if(a.content&&a.content.length>100)return a}finally{await pt(i)}}catch{}return process.stderr.write(`[greedysearch] HTTP failed for ${t.slice(0,60)}, trying browser...
|
|
371
|
+
`),await ra(t,e)}async function wr(t,e=4000,r=200){let n=Date.now()+e;while(Date.now()<n){try{if((await R(["eval",t,'document.readyState === "complete" && !!document.body && document.body.innerText.length > 500'])).trim()==="true")return}catch{}await new Promise((i)=>setTimeout(i,r))}}async function ra(t,e=8000){let r=Date.now(),n;try{n=await ft()}catch(i){return{url:t,title:"",content:null,snippet:"",contentChars:0,error:`openNewTab failed: ${i.message}`,source:"browser",duration:Date.now()-r}}try{await R(["nav",n,t],30000),await wr(n);let i=await R(["eval",n,String.raw`
|
|
372
|
+
(function(){
|
|
373
|
+
var el = document.querySelector('article, [role="main"], main, .post-content, .article-body, #content, .content');
|
|
374
|
+
var text = (el || document.body).innerText;
|
|
375
|
+
return JSON.stringify({
|
|
376
|
+
title: document.title,
|
|
377
|
+
content: text.replace(/\s+/g, ' ').trim(),
|
|
378
|
+
url: location.href
|
|
379
|
+
});
|
|
380
|
+
})()
|
|
381
|
+
`]),a=JSON.parse(i),o=re(a.content,e);return{url:t,finalUrl:a.url||t,status:200,contentType:"text/plain",lastModified:"",title:a.title,snippet:hr(o,320),content:o,contentChars:o.length,source:"browser",duration:Date.now()-r}}catch(i){return{url:t,title:"",content:null,snippet:"",contentChars:0,error:i.message,source:"browser",duration:Date.now()-r}}finally{await pt(n)}}async function na(t,e=5,r=8000,n=at){let i=t.slice(0,e);if(i.length===0)return[];let a=Math.min(i.length,Math.max(1,Number.parseInt(String(n),10)||at));process.stderr.write(`[greedysearch] Fetching content from ${i.length} sources via HTTP (concurrency ${a})...
|
|
382
|
+
`);let o=Array(i.length),s=0,l=0;async function c(){while(!0){let f=s++;if(f>=i.length)return;let m=i[f],g=m.canonicalUrl||m.url;process.stderr.write(`[greedysearch] [${f+1}/${i.length}] Fetching: ${g.slice(0,60)}...
|
|
383
|
+
`);let h=await yr(g,r).catch((y)=>({url:g,title:"",content:null,snippet:"",contentChars:0,error:y.message,source:"error",duration:0}));if(o[f]={id:m.id,...h},h.content&&h.content.length>100)process.stderr.write(`[greedysearch] ✓ ${h.source}: ${h.content.length} chars
|
|
384
|
+
`);else if(h.error)process.stderr.write(`[greedysearch] ✗ ${h.error.slice(0,80)}
|
|
385
|
+
`);l+=1,process.stderr.write(`PROGRESS:fetch:${l}/${i.length}
|
|
386
|
+
`)}}await Promise.all(Array.from({length:a},()=>c()));let u=o.filter((f)=>f.content&&f.content.length>100),d=o.filter((f)=>f.source==="http").length,p=o.filter((f)=>f.source==="browser").length;return process.stderr.write(`[greedysearch] Fetched ${u.length}/${o.length} sources (HTTP: ${d}, Browser: ${p})
|
|
387
|
+
`),o}async function ia(t){let e=await ft();try{await R(["nav",e,t],30000),await wr(e);let r=await R(["eval",e,String.raw`
|
|
388
|
+
(function(){
|
|
389
|
+
var el = document.querySelector('article, [role="main"], main, .post-content, .article-body, #content, .content');
|
|
390
|
+
var text = (el || document.body).innerText;
|
|
391
|
+
return text.replace(/\s+/g, ' ').trim();
|
|
392
|
+
})()
|
|
393
|
+
`]);return{url:t,content:r}}catch(r){return{url:t,content:null,error:r.message}}finally{await pt(e)}}var aa,oa,me=null,br="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",gt,ht,vr,$r,_r,Er,Sr,Rr,mt,Dr,Vt,Ar,Cr,Kt,sa,O,la,ca,ua,da,fa,pa,ga,ha,ma,Se,L,Re,Xe="gemini",et,we,ya,wa,ba,zt,va,N,$a,_a,Ea,Sa,Ra,Da,Aa,Ca,ka,De,M,Ae,tt="gemini",rt,be,Oa,Na,xa,Pa,Ia,ja,Ta,R,Zt,La,x,Ma,Ua,Ga,Fa,Ha,qa,Ja,Ya,Wa,Ce,U,ke,nt="gemini",it,ve,Ba,Qa,at,Va=dt(()=>{aa=ni(import.meta.url),oa=ii(import.meta.url),gt={"user-agent":br,accept:"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8","accept-language":"en-US,en;q=0.9","accept-encoding":"gzip, deflate, br","cache-control":"no-cache",pragma:"no-cache","sec-ch-ua":'"Chromium";v="122", "Not(A:Brand";v="24", "Google Chrome";v="122"',"sec-ch-ua-mobile":"?0","sec-ch-ua-platform":'"Windows"',"sec-fetch-dest":"document","sec-fetch-mode":"navigate","sec-fetch-site":"none","sec-fetch-user":"?1","upgrade-insecure-requests":"1"},ht=[/^localhost$/i,/^127\.\d+\.\d+\.\d+$/,/^0\.0\.0\.0$/,/^\[::1\]$/,/^10\./,/^172\.(1[6-9]|2\d|3[01])\./,/^192\.168\./,/^169\.254\./,/^fc00:/i,/^fe80:/i,/\.local$/i,/\.internal$/i,/\.localhost$/i],vr=/^::ffff:(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/i,$r=/^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/i,_r=/^(?:0x[0-9a-f]+|0[0-7]*|[1-9][0-9]*)$/i,Er=["accounts.google.com","login.microsoftonline.com","login.live.com","auth0.com","okta.com","auth.mozilla.auth0.com","id.atlassian.com"],Sr=["login.","signin.","auth.","sso.","accounts.","idp."],Rr=["sign in to continue","log in to continue","authentication required","create an account to continue","subscribe to continue reading","members only"],mt={"user-agent":"GreedySearch/1.0",accept:"application/vnd.github+json","x-github-api-version":"2022-11-28"},Dr={"user-agent":"GreedySearch/1.0 (Research Bot)",accept:"application/json"},Vt=si(ci(import.meta.url)),Ar=li(Vt,"..","bin","cdp.mjs"),Cr=new Set(["list","snap","eval","shot","html","nav","net","click","clickxy","type","loadall","evalraw","browse","stop","--tab"]),Kt=pi().replaceAll("\\","/"),sa=Ui(process.env.GREEDY_SEARCH_PORT),O=(process.env.GREEDY_SEARCH_PROFILE_DIR||process.env.CDP_PROFILE_DIR||`${Kt}/greedysearch-chrome-profile`).replaceAll("\\","/"),la=`${O}/DevToolsActivePort`,ca=process.env.GREEDY_SEARCH_PID_FILE||`${O}/browser.pid`,ua=process.env.CDP_PAGES_CACHE||`${O}/cdp-pages.json`,da=process.env.GREEDY_SEARCH_MODE_FILE||`${O}/browser-mode`,fa=process.env.GREEDY_SEARCH_METADATA_FILE||`${O}/browser-metadata.json`,pa=process.env.GREEDY_SEARCH_LAUNCH_LOCK_FILE||`${O}/browser-launch.lock`,ga=process.env.GREEDY_SEARCH_ACTIVITY_FILE||`${O}/browser-last-activity`,ha=(process.env.CDP_SOCKET_DIR||`${O}/cdp-sockets`).replaceAll("\\","/"),ma=`${O}/visible-recovery.jsonl`,Se=Gt(fi(),".dm"),L=Gt(Se,"greedyconfig"),Re=["perplexity","google","chatgpt","gemini"],Fi(),et=["gemini","chatgpt"],we={perplexity:"perplexity.mjs",p:"perplexity.mjs",bing:"bing-copilot.mjs",b:"bing-copilot.mjs",google:"google-ai.mjs",g:"google-ai.mjs",gemini:"gemini.mjs",gem:"gemini.mjs",chatgpt:"chatgpt.mjs",gpt:"chatgpt.mjs","semantic-scholar":"semantic-scholar.mjs",semanticscholar:"semantic-scholar.mjs",s2:"semantic-scholar.mjs",logically:"logically.mjs",log:"logically.mjs"},ya=Gi(),wa=Hi(),ba=Math.max(1,Number.parseInt(process.env.GREEDY_FETCH_CONCURRENCY||"5",10)||5),process.env.CDP_PROFILE_DIR=O,zt=yi().replaceAll("\\","/"),va=qi(process.env.GREEDY_SEARCH_PORT),N=(process.env.GREEDY_SEARCH_PROFILE_DIR||process.env.CDP_PROFILE_DIR||`${zt}/greedysearch-chrome-profile`).replaceAll("\\","/"),$a=`${N}/DevToolsActivePort`,_a=process.env.GREEDY_SEARCH_PID_FILE||`${N}/browser.pid`,Ea=process.env.CDP_PAGES_CACHE||`${N}/cdp-pages.json`,Sa=process.env.GREEDY_SEARCH_MODE_FILE||`${N}/browser-mode`,Ra=process.env.GREEDY_SEARCH_METADATA_FILE||`${N}/browser-metadata.json`,Da=process.env.GREEDY_SEARCH_LAUNCH_LOCK_FILE||`${N}/browser-launch.lock`,Aa=process.env.GREEDY_SEARCH_ACTIVITY_FILE||`${N}/browser-last-activity`,Ca=(process.env.CDP_SOCKET_DIR||`${N}/cdp-sockets`).replaceAll("\\","/"),ka=`${N}/visible-recovery.jsonl`,De=Ft(mi(),".dm"),M=Ft(De,"greedyconfig"),Ae=["perplexity","google","chatgpt","gemini"],Yi(),rt=["gemini","chatgpt"],be={perplexity:"perplexity.mjs",p:"perplexity.mjs",bing:"bing-copilot.mjs",b:"bing-copilot.mjs",google:"google-ai.mjs",g:"google-ai.mjs",gemini:"gemini.mjs",gem:"gemini.mjs",chatgpt:"chatgpt.mjs",gpt:"chatgpt.mjs","semantic-scholar":"semantic-scholar.mjs",semanticscholar:"semantic-scholar.mjs",s2:"semantic-scholar.mjs",logically:"logically.mjs",log:"logically.mjs"},Oa=Ji(),Na=Wi(),xa=Math.max(1,Number.parseInt(process.env.GREEDY_FETCH_CONCURRENCY||"5",10)||5),process.env.CDP_PROFILE_DIR=N,Pa=Number.parseInt(process.env.GREEDY_SEARCH_IDLE_TIMEOUT_MINUTES||"5",10)||5,Ia=import.meta.dirname||new URL(".",import.meta.url).pathname.replace(/^\/([A-Z]:)/,"$1"),ja=Number.parseInt(process.env.GREEDY_SEARCH_IDLE_TIMEOUT_MINUTES||"5",10)||5,Ta=Number.parseInt(process.env.GREEDY_SEARCH_VISIBLE_IDLE_TIMEOUT_MINUTES||"60",10)||60,R=pr,Zt=$i().replaceAll("\\","/"),La=Bi(process.env.GREEDY_SEARCH_PORT),x=(process.env.GREEDY_SEARCH_PROFILE_DIR||process.env.CDP_PROFILE_DIR||`${Zt}/greedysearch-chrome-profile`).replaceAll("\\","/"),Ma=`${x}/DevToolsActivePort`,Ua=process.env.GREEDY_SEARCH_PID_FILE||`${x}/browser.pid`,Ga=process.env.CDP_PAGES_CACHE||`${x}/cdp-pages.json`,Fa=process.env.GREEDY_SEARCH_MODE_FILE||`${x}/browser-mode`,Ha=process.env.GREEDY_SEARCH_METADATA_FILE||`${x}/browser-metadata.json`,qa=process.env.GREEDY_SEARCH_LAUNCH_LOCK_FILE||`${x}/browser-launch.lock`,Ja=process.env.GREEDY_SEARCH_ACTIVITY_FILE||`${x}/browser-last-activity`,Ya=(process.env.CDP_SOCKET_DIR||`${x}/cdp-sockets`).replaceAll("\\","/"),Wa=`${x}/visible-recovery.jsonl`,Ce=Ht(vi(),".dm"),U=Ht(Ce,"greedyconfig"),ke=["perplexity","google","chatgpt","gemini"],Vi(),it=["gemini","chatgpt"],ve={perplexity:"perplexity.mjs",p:"perplexity.mjs",bing:"bing-copilot.mjs",b:"bing-copilot.mjs",google:"google-ai.mjs",g:"google-ai.mjs",gemini:"gemini.mjs",gem:"gemini.mjs",chatgpt:"chatgpt.mjs",gpt:"chatgpt.mjs","semantic-scholar":"semantic-scholar.mjs",semanticscholar:"semantic-scholar.mjs",s2:"semantic-scholar.mjs",logically:"logically.mjs",log:"logically.mjs"},Ba=Qi(),Qa=Ki(),at=Math.max(1,Number.parseInt(process.env.GREEDY_FETCH_CONCURRENCY||"5",10)||5),process.env.CDP_PROFILE_DIR=x}),Ka={};ut(Ka,{parseGitHubUrl:()=>kr,fetchGitHubContent:()=>Xa});function kr(t){try{let e=new URL(t);if(!(e.hostname==="github.com"||e.hostname.endsWith(".github.com")))return null;let r=e.pathname.split("/").filter(Boolean);if(r.length<2)return null;let[n,i]=r;if(r.length===2)return{owner:n,repo:i,type:"root"};if(r.length>=4&&(r[2]==="blob"||r[2]==="tree")){let a=r[2],o=r[3],s=r.slice(4).join("/");return{owner:n,repo:i,type:a,ref:o,path:s}}return null}catch{return null}}async function I(t,e=1e4){let r=new AbortController,n=setTimeout(()=>r.abort(),e);try{let i=await fetch(`https://api.github.com${t}`,{headers:yt,signal:r.signal});if(clearTimeout(n),!i.ok)throw Error(`GitHub API ${i.status}: ${t}`);return await i.json()}catch(i){throw clearTimeout(n),i}}async function za(t,e){try{let r=await I(`/repos/${t}/${e}/readme`);if(r.content&&r.encoding==="base64")return Buffer.from(r.content,"base64").toString("utf8");return""}catch{return""}}async function Xt(t,e,r="HEAD",n="",i){try{let a;if(r==="HEAD")if(i)a=await I(`/repos/${t}/${e}/git/ref/heads/${i}`).catch(()=>null);else a=await Promise.any([I(`/repos/${t}/${e}/git/ref/heads/main`),I(`/repos/${t}/${e}/git/ref/heads/master`)]).catch(()=>null);else a=await I(`/repos/${t}/${e}/git/ref/heads/${r}`).catch(()=>I(`/repos/${t}/${e}/git/ref/heads/master`).catch(()=>null));if(!a?.object?.sha)return[];let o=(await I(`/repos/${t}/${e}/git/commits/${a.object.sha}`)).tree.sha,s=(await I(`/repos/${t}/${e}/git/trees/${o}`)).tree||[];if(n)s=s.filter((l)=>l.path.startsWith(n));return s.slice(0,50).map((l)=>({path:l.path,type:l.type==="tree"?"dir":"file",size:l.size}))}catch{return[]}}async function Za(t,e,r,n,i=1e4,a){let o=async(l)=>{let c=new AbortController,u=setTimeout(()=>c.abort(),i);try{let d=await fetch(l,{headers:{"user-agent":yt["user-agent"]},signal:c.signal});if(clearTimeout(u),d.ok)return await d.text();throw Error("not ok")}catch{throw clearTimeout(u),Error("failed")}};if(!r||r==="HEAD"){if(a)try{return await o(`https://raw.githubusercontent.com/${t}/${e}/${a}/${n}`)}catch{return null}try{return await Promise.any([o(`https://raw.githubusercontent.com/${t}/${e}/main/${n}`),o(`https://raw.githubusercontent.com/${t}/${e}/master/${n}`)])}catch{return null}}let s=[`https://raw.githubusercontent.com/${t}/${e}/${r}/${n}`,`https://raw.githubusercontent.com/${t}/${e}/master/${n}`];for(let l of s)try{return await o(l)}catch{}return null}async function Xa(t){let e=kr(t);if(!e)return{ok:!1,error:"Not a valid GitHub URL"};let{owner:r,repo:n,type:i,ref:a,path:o}=e;try{if(i==="root"||i==="tree"&&!o){let s=await I(`/repos/${r}/${n}`),[l,c]=await Promise.allSettled([za(r,n),Xt(r,n,a||"HEAD","",s.default_branch)]),u=l.status==="fulfilled"?l.value:"",d=c.status==="fulfilled"?c.value:[],p=s?.description?`
|
|
394
|
+
|
|
395
|
+
> ${s.description}`:"",f=s?.stargazers_count==null?"":` ⭐ ${s.stargazers_count}`,m=s?.language?` · ${s.language}`:"",g=`# ${r}/${n}${f}${m}${p}
|
|
396
|
+
|
|
397
|
+
`;if(u)g+=u.slice(0,6000);else g+=`[No README found]
|
|
398
|
+
|
|
399
|
+
Files:
|
|
400
|
+
${d.map((h)=>` ${h.type==="dir"?"\uD83D\uDCC1":"\uD83D\uDCC4"} ${h.path}`).join(`
|
|
401
|
+
`)}`;return{ok:!0,title:`${r}/${n}`,content:g,tree:d.slice(0,30)}}if(i==="blob"&&o){let s;if(!a||a==="HEAD")try{s=(await I(`/repos/${r}/${n}`)).default_branch}catch{s=void 0}let l=await Za(r,n,a,o,1e4,s);if(l===null)return{ok:!1,error:`File not found: ${o}`};return{ok:!0,title:`${r}/${n}: ${o}`,content:l}}if(i==="tree"&&o){let s;if(!a||a==="HEAD")try{s=(await I(`/repos/${r}/${n}`)).default_branch}catch{s=void 0}let l=await Xt(r,n,a||"HEAD",o,s),c=l.map((u)=>` ${u.type==="dir"?"\uD83D\uDCC1":"\uD83D\uDCC4"} ${u.path}`).join(`
|
|
402
|
+
`);return{ok:!0,title:`${r}/${n}/${o}`,content:`[Directory: ${o}]
|
|
403
|
+
|
|
404
|
+
Files:
|
|
405
|
+
${c}`,tree:l}}return{ok:!1,error:"Unsupported GitHub URL type"}}catch(s){return{ok:!1,error:s.message}}}var yt,Ec=dt(()=>{yt={"user-agent":"GreedySearch/1.0",accept:"application/vnd.github+json","x-github-api-version":"2022-11-28"}});function to(t=process.env,e=process.execPath){let r=t.GREEDY_SEARCH_NODE||t.NODE_BINARY||t.NODE;if(r?.trim())return r.trim();let n=eo(e||"").toLowerCase();if(n==="node"||n==="node.exe")return e;return"node"}var no=["fbclid","gclid","ref","ref_src","ref_url","source","utm_campaign","utm_content","utm_medium","utm_source","utm_term"];function z(t="",e=240){let r=String(t).replaceAll(/\s+/g," ").trim();if(r.length<=e)return r;let n=r.slice(0,e),i=n.lastIndexOf(" ");return i>0?`${n.slice(0,i)}...`:`${n}...`}function ot(t){if(!t)return null;try{let e=new URL(t);if(!["http:","https:"].includes(e.protocol))return null;if(e.hash="",e.hostname=e.hostname.toLowerCase(),e.protocol==="https:"&&e.port==="443"||e.protocol==="http:"&&e.port==="80")e.port="";for(let i of[...e.searchParams.keys()]){let a=i.toLowerCase();if(no.includes(a)||a.startsWith("utm_"))e.searchParams.delete(i)}e.searchParams.sort();let r=e.pathname.replace(/\/{1,10}$/,"")||"/";e.pathname=r;let n=e.toString();return r==="/"?n.replace(/\/$/,""):n}catch{return null}}function lo(t,e=9222){let r=Number.parseInt(String(t??""),10);return Number.isInteger(r)&&r>1024&&r<65535?r:e}var co=so().replaceAll("\\","/"),xc=lo(process.env.GREEDY_SEARCH_PORT),j=(process.env.GREEDY_SEARCH_PROFILE_DIR||process.env.CDP_PROFILE_DIR||`${co}/greedysearch-chrome-profile`).replaceAll("\\","/"),Pc=`${j}/DevToolsActivePort`,Ic=process.env.GREEDY_SEARCH_PID_FILE||`${j}/browser.pid`,jc=process.env.CDP_PAGES_CACHE||`${j}/cdp-pages.json`,Tc=process.env.GREEDY_SEARCH_MODE_FILE||`${j}/browser-mode`,Lc=process.env.GREEDY_SEARCH_METADATA_FILE||`${j}/browser-metadata.json`,Mc=process.env.GREEDY_SEARCH_LAUNCH_LOCK_FILE||`${j}/browser-launch.lock`,Uc=process.env.GREEDY_SEARCH_ACTIVITY_FILE||`${j}/browser-last-activity`,Gc=(process.env.CDP_SOCKET_DIR||`${j}/cdp-sockets`).replaceAll("\\","/"),Fc=`${j}/visible-recovery.jsonl`,st=Nr(oo(),".dm"),J=Nr(st,"greedyconfig"),lt=["perplexity","google","chatgpt","gemini"],ct="gemini";function uo(){try{if(Oe(J)){let t=Or(J,"utf8"),e=JSON.parse(t);if(Array.isArray(e.engines)&&e.engines.length>0&&e.engines.every((r)=>typeof r==="string")){let r=e.engines.filter((i)=>Ve[i]),n=e.engines.filter((i)=>!Ve[i]);if(n.length>0)process.stderr.write(`[greedysearch] Warning: ignoring unknown engine(s) in ${J}: ${n.join(", ")}
|
|
406
|
+
[greedysearch] Available engines: ${Object.keys(Ve).join(", ")}
|
|
407
|
+
`);if(r.length>0)return r;process.stderr.write(`[greedysearch] Warning: no valid engines in ${J}, falling back to defaults: ${lt.join(", ")}
|
|
408
|
+
`)}}}catch{}return lt}function fo(){try{if(!Oe(st))io(st,{recursive:!0});if(!Oe(J))ao(J,JSON.stringify({engines:lt,synthesizer:ct},null,2)+`
|
|
409
|
+
`,"utf8")}catch{}}fo();var Ne=["gemini","chatgpt"];function po(){try{if(Oe(J)){let t=Or(J,"utf8"),e=JSON.parse(t);if(typeof e.synthesizer==="string"){let r=e.synthesizer.toLowerCase();if(Ne.includes(r))return r;process.stderr.write(`[greedysearch] Warning: unknown synthesizer "${e.synthesizer}" in ${J}
|
|
410
|
+
[greedysearch] Available synthesizers: ${Ne.join(", ")}
|
|
411
|
+
[greedysearch] Falling back to default: ${ct}
|
|
412
|
+
`)}}}catch{}return ct}var Ve={perplexity:"perplexity.mjs",p:"perplexity.mjs",bing:"bing-copilot.mjs",b:"bing-copilot.mjs",google:"google-ai.mjs",g:"google-ai.mjs",gemini:"gemini.mjs",gem:"gemini.mjs",chatgpt:"chatgpt.mjs",gpt:"chatgpt.mjs","semantic-scholar":"semantic-scholar.mjs",semanticscholar:"semantic-scholar.mjs",s2:"semantic-scholar.mjs",logically:"logically.mjs",log:"logically.mjs"},xr=uo();var Hc=po(),qc=Math.max(1,Number.parseInt(process.env.GREEDY_FETCH_CONCURRENCY||"5",10)||5);process.env.CDP_PROFILE_DIR=j;function go(t){let e="",r=!1,n=!1;for(let i of String(t)){if(n){e+=i,n=!1;continue}if(i==="\\"){e+=i,n=!0;continue}if(i==='"'){r=!r,e+=i;continue}if(r&&i===`
|
|
413
|
+
`)e+="\\n";else if(r&&i==="\r")e+="\\r";else if(r&&i==="\t")e+="\\t";else e+=i}return e}function ho(t){if(!t)return null;let e=String(t).trim(),r=e.indexOf("BEGIN_JSON"),n=e.indexOf("END_JSON");if(r!==-1&&n!==-1&&r<n)e=e.slice(r+10,n).trim();else{let s=e.indexOf("{");if(s>0)e=e.slice(s)}let i=[e,e.replace(/^```json\s*/i,"").replace(/^```\s*/i,"").replace(/```$/i,"").trim()],a=e.indexOf("{"),o=e.lastIndexOf("}");if(a!==-1&&o!==-1&&a<o)i.push(e.slice(a,o+1));for(let s of[...i]){let l=go(s);if(l!==s)i.push(l)}for(let s of i)try{return JSON.parse(s)}catch{}return null}function bo(t){return wo(new URL(".",t)).replace(/^\/([A-Z]:)/,"$1")}function vo(t,e=Pr){let r=new Set;for(let n of t.filter(Boolean)){if(r.has(n))continue;if(r.add(n),e(n))return n}return t.filter(Boolean).at(-1)}function $o(t,{moduleDir:e,entrypoint:r=process.argv[1],env:n=process.env,exists:i=Pr}={}){let a=r?yo(r):null,o=n.GREEDY_SEARCH_EXTENSION_DIR?.trim()||null;return vo([e?ye(e,"..","..","extractors",t):null,e?ye(e,"..","extractors",t):null,o?ye(o,"extractors",t):null,a?ye(a,"..","extractors",t):null],i)}var _o=bo(import.meta.url),Eo={gemini:"gemini.mjs",chatgpt:"chatgpt.mjs"};function So(t="gemini"){let e=String(t||"gemini").toLowerCase();if(e==="gem")return"gemini";if(e==="gpt")return"chatgpt";return e}async function Ro(t,e,{tabPrefix:r=null,timeoutMs:n=180000,visible:i=null}={}){let a=So(t),o=Eo[a];if(!o||!Ne.includes(a))throw Error(`Unsupported synthesizer "${t}". Supported: ${Ne.join(", ")}`);return new Promise((s,l)=>{let c=$o(o,{moduleDir:_o}),u=r?["--tab",String(r)]:[],d={...process.env,CDP_PROFILE_DIR:j};if(i!==!0)delete d.GREEDY_SEARCH_VISIBLE,delete d.GREEDY_SEARCH_ALWAYS_VISIBLE;else d.GREEDY_SEARCH_VISIBLE="1",d.GREEDY_SEARCH_ALWAYS_VISIBLE="1";let p=mo(to(),[c,"--stdin",...u],{stdio:["pipe","pipe","pipe"],env:d});p.stdin.write(e),p.stdin.end();let f="",m="";p.stdout.on("data",(h)=>f+=h),p.stderr.on("data",(h)=>m+=h);let g=setTimeout(()=>{p.kill(),l(Error(`${a} prompt timed out after ${n/1000}s`))},n);p.on("close",(h)=>{if(clearTimeout(g),h!==0){l(Error(m.trim()||`${a} extractor failed`));return}try{s(JSON.parse(f.trim()))}catch{l(Error(`bad JSON from ${a}: ${f.slice(0,100)}`))}})})}async function Do(t,e={}){return Ro("gemini",t,e)}ar();Va();var ko=Co(new URL(".",import.meta.url)).replace(/^\/([A-Z]:)/,"$1"),Kc=Ao(ko,"..","..","bin","search.mjs");var Oo=xr.join("|"),zc=new RegExp(`^\\[(${Oo})\\]`);var No=ro(new URL(".",import.meta.url)).replace(/^\/([A-Z]:)/,"$1"),Zc=_(No,"..","..","bin","search.mjs"),xo=_(process.cwd(),".pi","greedysearch-research");function Po(t){return String(t||"research").toLowerCase().replaceAll(/[^a-z0-9]+/g,"-").replaceAll(/^-|-$/g,"").slice(0,60)||"research"}function se(t,e=1/0){let r=new Set,n=[];for(let i of t||[]){let a=z(String(i||""),1000);if(!a||r.has(a))continue;if(r.add(a),n.push(a),n.length>=e)break}return n}async function Io(...t){let{writeSourcesToFiles:e}=await Promise.resolve().then(()=>(ar(),rr));return e(...t)}function Ir(t){return To(jo(String(t)))}function jo(t){let e="",r=0;while(r<t.length){let n=t.indexOf("[",r);if(n===-1){e+=t.slice(r);break}let i=t.indexOf("]",n+1);if(i===-1||t[i+1]!=="("||i===n+1){e+=t.slice(r,n+1),r=n+1;continue}let a=t.indexOf(")",i+2);if(a===-1){e+=t.slice(r,n+1),r=n+1;continue}let o=t.slice(i+2,a).trimStart();if(!o.startsWith("http://")&&!o.startsWith("https://")){e+=t.slice(r,n+1),r=n+1;continue}e+=t.slice(r,n),e+=t.slice(n+1,i),r=a+1}return e}function To(t){let e="",r=!1;for(let n of t)if(n===" "||n==="\t"||n===`
|
|
414
|
+
`||n==="\r"){if(!r)e+=" ";r=!0}else e+=n,r=!1;return e.trim()}function er(t){return new Set(String(t).toLowerCase().normalize("NFD").replaceAll(/[\u0300-\u036f]/g,"").split(/[^\w]+/).filter((e)=>e.length>1))}function jr(t,e){let r=er(t),n=er(e),i=new Set([...r,...n]).size;if(i===0)return 1;let a=0;for(let o of r)if(n.has(o))a++;return a/i}function xe(t){return ot(t?.finalUrl||t?.canonicalUrl||t?.url||"")||t?.id||""}function Lo(t,e){let r=Array.isArray(t?.extractions)?t.extractions:[],n=new Map,i=new Map;for(let a of e||[]){if(a?.id)i.set(String(a.id),a);let o=xe(a);if(o)n.set(o,a)}return r.map((a)=>{let o=i.get(String(a?.sourceId||""))||n.get(ot(a?.url||"")||""),s=String(a?.sourceId||o?.id||""),l=ot(a?.url||o?.finalUrl||o?.url||""),c=Array.isArray(a?.answers)?a.answers.map((u)=>({id:String(u?.id||""),evidence:z(u?.evidence||"",500),sourceIds:[s].filter(Boolean)})).filter((u)=>u.id):[];return{sourceId:s,url:l,title:o?.title||a?.title||"",rational:z(a?.rational||"",700),evidence:z(a?.evidence||"",1600),summary:z(a?.summary||"",700),answers:c,newQuestions:se(a?.newQuestions||[],6)}}).filter((a)=>a.sourceId||a.url||a.summary||a.evidence)}function Mo(t,e,r,n=new Set){let i=(e||[]).filter((o)=>o.status!=="closed").slice(0,12).map((o)=>({id:o.id,question:o.question})),a=(r||[]).filter((o)=>o?.content||o?.snippet).filter((o)=>!n.has(xe(o))).slice(0,6).map((o,s)=>({id:o.id||`F${s+1}`,title:o.title||"",url:o.finalUrl||o.url||o.canonicalUrl||"",content:z(o.content||o.snippet||"",5000)}));return["You are doing goal-based evidence extraction for an iterative research run.","For each source, extract only information that helps answer the open questions.","Use original wording/details where useful. Do not invent answers; leave questions open if evidence is insufficient.","If a source answers one or more tracked questions, identify those question IDs explicitly.","Also propose genuinely new sub-questions discovered from the evidence.","",`Original research question: ${t}`,`Open question ledger: ${JSON.stringify(i,null,2)}`,`Fetched sources: ${JSON.stringify(a,null,2)}`,"","Respond ONLY with JSON wrapped in BEGIN_JSON / END_JSON markers:","BEGIN_JSON",JSON.stringify({extractions:[{sourceId:"S1",url:"https://example.com/source",rational:"why this source matters for the goal",evidence:"specific quoted/paraphrased evidence with numbers, dates, caveats",summary:"concise contribution to the research question",answers:[{id:"Q1",evidence:"brief evidence that closes the question"}],newQuestions:["new sub-question raised by this source"]}]},null,2),"END_JSON"].join(`
|
|
415
|
+
`)}async function Tr({query:t,questions:e,fetchedSources:r,extractedSourceKeys:n}){let i=(r||[]).filter((a)=>(a?.content||a?.snippet)&&!n.has(xe(a)));if(i.length===0)return{evidence:[],error:""};try{let a=await Do(Mo(t,e,i,n),{timeoutMs:120000}),o=Go(a,{extractions:[]}),s=Lo(o,i);for(let l of i){let c=xe(l);if(c)n.add(c)}return{evidence:s,error:""}}catch(a){return{evidence:[],error:a.message||String(a)}}}function Lr(t,e,r,n=[],i=[]){let a=e.flatMap((l)=>l.learnings||[]),o=e.flatMap((l)=>l.gaps||[]),s=r.slice(0,12).map((l)=>({id:l.id,title:l.title,domain:l.domain,url:l.canonicalUrl,type:l.sourceType,engines:l.engines,fetch:l.fetch?.attempted?{ok:l.fetch.ok,snippet:z(l.fetch.snippet||"",1200),publishedTime:l.fetch.publishedTime||""}:void 0}));return["You are writing the final research report for an iterative deep-research run.","Produce a thorough markdown report organized into clear sections.","","Use the learnings and source registry below. Every substantive claim MUST be backed by an [S1] citation.",'Where engines disagree, surface the conflicting claims explicitly in the "differences" array.','Include a "Key Claims" structure that maps each distinct claim to its supporting source IDs.',"","Report structure:","1. ## Summary — A 2-4 sentence executive summary of findings","2. ## Key Findings — The main findings, organized by theme or question, each with inline citations","3. ## Areas of Disagreement — Where engines or sources conflict (if any)","4. ## Limitations & Caveats — Important qualifiers, gaps, or uncertainties","",`Original research question: ${t}`,`Learnings: ${JSON.stringify(a,null,2)}`,`Known gaps/caveats: ${JSON.stringify(o,null,2)}`,`Question ledger: ${JSON.stringify(n,null,2)}`,`Goal-based extracted evidence: ${JSON.stringify(i.slice(-20),null,2)}`,`Source registry: ${JSON.stringify(s,null,2)}`,"","Respond ONLY with JSON wrapped in BEGIN_JSON / END_JSON markers:","BEGIN_JSON",JSON.stringify({answer:"markdown report with sections and inline [S1] citations",agreement:{level:"high|medium|low|mixed|conflicting",summary:"one-sentence confidence summary"},differences:["notable disagreement or conflict between sources"],caveats:["important caveat or qualification"],claims:[{claim:"specific factual statement from the research",support:"strong|moderate|weak|conflicting",sourceIds:["S1","S2"]}],recommendedSources:["S1","S2"]},null,2),"END_JSON"].join(`
|
|
416
|
+
`)}function Mr(t,e=[],r=[],n=[]){let i=e.slice(0,12).map((l)=>({id:l.id,title:l.title,domain:l.domain,url:l.canonicalUrl,type:l.sourceType,engines:l.engines})),a=n.slice(-20),o=new Set;for(let l of a)for(let c of l.answers||[])if(c?.id)o.add(c.id);let s=(r||[]).filter((l)=>l.status!=="closed").map((l)=>({id:l.id,question:l.question}));return["You are writing the final research report from goal-based extracted evidence.","Per-round learnings were not produced, but the per-source evidence extraction step succeeded.","Synthesize a thorough markdown report using ONLY the evidence below. Every substantive claim MUST be backed by an [S1] citation.","","Report structure:","1. ## Summary — A 2-4 sentence executive summary of findings","2. ## Key Findings — The main findings, organized by theme or question, each with inline citations","3. ## Limitations & Caveats — Important qualifiers, gaps, or uncertainties","",`Original research question: ${t}`,`Per-source extracted evidence: ${JSON.stringify(a,null,2)}`,`Source registry: ${JSON.stringify(i,null,2)}`,`Questions already answered by the evidence: ${JSON.stringify(Array.from(o))}`,`Questions still open after this evidence: ${JSON.stringify(s)}`,"","Respond ONLY with JSON wrapped in BEGIN_JSON / END_JSON markers:","BEGIN_JSON",JSON.stringify({answer:"markdown report with sections and inline [S1] citations",agreement:{level:"high|medium|low|mixed|conflicting",summary:"one-sentence confidence summary"},differences:["notable disagreement or conflict between sources"],caveats:["important caveat or qualification"],claims:[{claim:"specific factual statement supported by the evidence",support:"strong|moderate|weak|conflicting",sourceIds:["S1","S2"]}],recommendedSources:["S1","S2"]},null,2),"END_JSON"].join(`
|
|
417
|
+
`)}var Uo=xr.join("|"),Xc=new RegExp(`^\\[(${Uo})\\]`);function Go(t,e={}){return ho(t?.answer||"")||e}function Ur(t,e){if(!t||!Array.isArray(e))return{cited:[],missing:[],unfetched:[],ok:!0};let r=/\b[SF](\d+)\b/g,n=new Set,i;while((i=r.exec(t))!==null)n.add(`S${i[1]}`),n.add(`F${i[1]}`);let a=new Map;for(let c of e){let u=c?.id;if(u)a.set(u,c)}let o=Array.from(n),s=[],l=[];for(let c of o){let u=a.get(c);if(!u){let d=c.match(/^(S|F)(\d+)$/);if(d){let p=parseInt(d[2],10)-1;if(p>=0&&p<e.length){let f=e[p];if(f){if(!(f.fetch?.ok||f.content&&f.content.length>100||f.contentChars&&f.contentChars>100))l.push(c);continue}}}s.push(c)}else if(!(u.fetch?.ok||u.content&&u.content.length>100||u.contentChars&&u.contentChars>100))l.push(c)}return{cited:o,missing:s,unfetched:l,ok:s.length===0}}async function Fo(t,{timeoutMs:e=6000,concurrency:r=4}={}){let n=Math.max(1,Math.floor(r||1)),i=(t||[]).filter((p)=>p?.id&&(p?.canonicalUrl||p?.finalUrl||p?.url));if(i.length===0)return{reachable:[],dead:[],skipped:[],ok:!0};let a=[],o=[],s=[],l=Array(i.length),c=0;async function u(){while(!0){let p=c++;if(p>=i.length)return;let f=i[p];try{let m=f.fetch?.finalUrl||f.canonicalUrl||f.finalUrl||f.url;if(!m){l[p]={id:f.id,url:"",status:"skipped"};continue}try{let g=new URL(m);if(g.protocol!=="http:"&&g.protocol!=="https:"){l[p]={id:f.id,url:m,status:"skipped"};continue}}catch{l[p]={id:f.id,url:m,status:"skipped"};continue}try{let g=new AbortController,h=setTimeout(()=>g.abort(),e);try{let y=await fetch(m,{method:"HEAD",redirect:"follow",signal:g.signal,headers:{"User-Agent":"Mozilla/5.0 (compatible; GreedySearch/2.0; +https://github.com/apmantza/greedysearch-dm)"}});clearTimeout(h);let v=y.status>=200&&y.status<400,b=[401,403,405,429].includes(y.status),w="dead";if(v)w="reachable";else if(b)w="skipped";l[p]={id:f.id,url:m,status:w,httpStatus:y.status,reason:b?"bot-protected-or-head-disallowed":void 0}}catch(y){clearTimeout(h),l[p]={id:f.id,url:m,status:"dead",error:y.name==="AbortError"?"timeout":y.message}}}catch(g){l[p]={id:f.id,url:m,status:"dead",error:g.message}}}catch(m){l[p]={id:"?",url:"",status:"dead",error:m?.message||"unknown"}}}}let d=Math.min(i.length,n);await Promise.all(Array.from({length:d},()=>u()));for(let p of l)if(p.status==="reachable")a.push(p);else if(p.status==="dead")o.push(p);else s.push(p);return{reachable:a,dead:o,skipped:s,ok:o.length===0}}async function Gr(t,e=null){process.stderr.write(`PROGRESS:research:check-urls
|
|
418
|
+
`);try{let r=new Set(e?.cited||[]),n=r.size?(t||[]).filter((a)=>r.has(a?.id)):t,i=await Fo(n,{timeoutMs:6000,concurrency:4});if(!i.ok)process.stderr.write(`[greedysearch] ${i.dead.length} dead citation URL(s) detected
|
|
419
|
+
`);return i}catch(r){return process.stderr.write(`[greedysearch] URL reachability check failed: ${r.message}
|
|
420
|
+
`),null}}function Fr({sources:t=[],fetchedSources:e=[],synthesis:r={},citationAudit:n=null,gaps:i=[],questions:a=[],rounds:o=[],qualityScore:s=0,qualityThreshold:l=8.5,maxSources:c=8,requireCitations:u=!0,requireQuestions:d=!0}={}){let p=e.filter((S)=>S?.fetch?.ok||(S?.contentChars||0)>100||String(S?.content||"").length>100),f=t.filter((S)=>["official-docs","repo","maintainer-blog","academic"].includes(String(S?.sourceType||""))),m=Array.isArray(r?.claims)?r.claims:[],g=n?n.cited?.length||0:0,h=tr(a),y=(a||[]).filter((S)=>!S.createdRound||S.reason==="Original research question"),v=tr(y),b=(o||[]).length,w=Math.min(4,Math.max(2,Number(c)||8)),E=b<=1?Math.min(2,w):w,A={roundsRun:o.length>=1,fetchedSources:p.length>=E,primarySources:f.length>=1,qualityScore:s>=Math.min(l,8)||u&&m.length>0&&g>0,claimsExtracted:!u||m.length>0,citationsPresent:!u||g>0,citationsValid:!u||n?.ok===!0,unfetchedCitations:!u||(n?.unfetched||[]).length===0,requiredQuestionsClosed:!d||v.open===0};return{floorMet:Object.values(A).every(Boolean),checks:A,metrics:{fetchedOk:p.length,primarySources:f.length,claims:m.length,cited:g,gaps:i.length,openQuestions:h.open,closedQuestions:h.closed,totalQuestions:h.total,openRequiredQuestions:v.open,closedRequiredQuestions:v.closed,totalRequiredQuestions:v.total,qualityScore:s,minFetched:E}}}function Hr(t){return[{id:"Q1",question:z(Ir(t),320),status:"open",reason:"Original research question",evidence:[],sourceIds:[]}]}function Ho(t,e){let r=Ir(e).toLowerCase();return(t||[]).find((n)=>n.question?.toLowerCase()===r||jr(n.question||"",r)>=0.82)}function qo(t,e,{evidence:r="",sourceIds:n=[],round:i=null}={}){let a=t.find((o)=>o.id===e)||Ho(t,e);if(!a)return null;if(a.status="closed",a.closedRound=a.closedRound||i,r)a.evidence=se([...a.evidence||[],r],4);if(Array.isArray(n))a.sourceIds=se([...a.sourceIds||[],...n],8);return a}function tr(t){let e=t.length,r=t.filter((n)=>n.status==="closed").length;return{total:e,closed:r,open:Math.max(0,e-r)}}function qr(t,e,r){if(!e?.answer||r?.ok!==!0)return t;let n=Array.isArray(e.claims)?e.claims:[],i=Array.isArray(r.cited)?r.cited:[];if(n.length===0||i.length===0)return t;for(let a of t){if(a.status==="closed")continue;let o=null,s=0;for(let l of n){let c=jr(a.question||"",l.claim||"");if(c>s)s=c,o=l}if(a.id==="Q1"||s>=0.18)qo(t,a.id,{evidence:o?.claim||"Answered in final cited synthesis",sourceIds:Array.isArray(o?.sourceIds)?o.sourceIds:i.slice(0,4)})}return t}function Jo(t){if(!t.length)return"No tracked questions.";return t.map((e)=>{let r=e.sourceIds?.length?` (${e.sourceIds.join(", ")})`:"";return`- [${e.status==="closed"?"x":" "}] ${e.id}: ${e.question}${r}`}).join(`
|
|
421
|
+
`)}function Ke(t,e="None recorded."){let r=se(t);return r.length?r.map((n)=>`- ${n}`).join(`
|
|
422
|
+
`):e}function Yo(t,{query:e,rounds:r,sources:n,fetchedSources:i,citationAudit:a,citationUrls:o,floor:s,manifest:l}){let c=(i||[]).filter((g)=>g?.contentChars>100||g?.fetch?.ok),u=(n||[]).filter((g)=>["official-docs","repo","maintainer-blog","academic"].includes(String(g?.sourceType||""))),d=new Set(a?.cited||[]),p=(n||[]).filter((g)=>d.has(g?.id)),f=[`# Provenance: ${e}`,"",`- **Date:** ${l?.startedAt||new Date().toISOString()}`,`- **Duration:** ${l?.durationMs?`${(l.durationMs/1000).toFixed(1)}s`:"unknown"}`,`- **Mode:** ${l?.terminationReason==="simple_single_pass"?"simple (single-pass)":"iterative"}`,`- **Rounds:** ${l?.rounds||r?.length||1}`,"","## Sources","",`- **Consulted:** ${n?.length||0}`,`- **Fetched successfully:** ${c.length}`,`- **Primary sources:** ${u.length}`,`- **Cited in report:** ${p.length}`,""];if(p.length>0){f.push("### Cited sources","");for(let g of p){let h=g.canonicalUrl||g.finalUrl||g.url||"",y=g.fetch?.ok?"✓":"✗";f.push(`- **${g.id}:** [${g.title||h}](${h}) (${g.sourceType||"unknown"}, fetched: ${y})`)}f.push("")}if(o&&(o.reachable.length>0||o.dead.length>0)){if(f.push("## URL reachability",""),o.dead.length>0){f.push(""),f.push("**Dead links:**");for(let g of o.dead)f.push(`- ${g.id}: ${g.url} (${g.httpStatus||g.error||"unknown"})`)}if(o.reachable.length>0)f.push(""),f.push(`**Reachable:** ${o.reachable.length}/${o.reachable.length+o.dead.length}`);f.push("")}let m=!a?"NOT CHECKED":a.ok&&(o?.ok??!0)?"PASS":a.ok===!1?"FAIL (missing citations)":"FAIL (dead links)";if(f.push("## Verification","",`- **Citations:** ${a?.ok?"PASS":`FAIL — missing: ${(a?.missing||[]).join(", ")}`}`,`- **URL reachability:** ${o?o.ok?"PASS":`FAIL — ${o.dead.length} dead`:"SKIPPED"}`,`- **Floor:** ${s?.floorMet?"PASS":"PARTIAL"}`,`- **Overall:** ${m}`,""),s?.checks){f.push("## Floor checks","");for(let[g,h]of Object.entries(s.checks))f.push(`- [${h?"x":" "}] ${g}`);f.push("")}C(_(t,"provenance.md"),f.join(`
|
|
423
|
+
`),"utf8")}async function Jr({query:t,rounds:e,sources:r,fetchedSources:n,evidenceItems:i=[],synthesis:a,citationAudit:o,floor:s,manifest:l,allGaps:c=[],questions:u=[],citationUrls:d=null,outDir:p=null}){let f=new Date().toISOString().replaceAll(/[:.]/g,"-").slice(0,19),m=p||_(xo,`${f}_${Po(t)}`),g=_(m,"reports"),h=_(m,"sources"),y=_(m,"data");Qe(g,{recursive:!0}),Qe(h,{recursive:!0}),Qe(y,{recursive:!0});let v=await Io(n,h),b=se([...c,...e.flatMap((w)=>w.gaps||[])]);C(_(m,"STATUS.md"),[s.floorMet?"STATUS: DONE":"STATUS: PARTIAL","",`Query: ${t}`,`Stop reason: ${l.terminationReason||"max_rounds"}`,"","## Deterministic floor checks",...Object.entries(s.checks).map(([w,E])=>`- [${E?"x":" "}] ${w}`),"","## Questions",Jo(u),"","## Open gaps",Ke(b),""].join(`
|
|
424
|
+
`),"utf8"),C(_(m,"OUTLINE.md"),["# Research bundle outline","","- `reports/SUMMARY.md` — final cited report","- `reports/CLAIMS.md` — extracted claims with support/source IDs","- `reports/EVIDENCE.md` — goal-based source evidence","- `reports/GAPS.md` — remaining caveats and uncertainties","- `provenance.md` — human-readable run metadata and verification","- `sources/` — fetched source markdown files","- `data/manifest.json` — machine-readable run metadata","- `data/rounds.json` — per-round actions/learnings/gaps","- `data/sources.json` — ranked source registry","- `data/questions.json` — open/closed question ledger",""].join(`
|
|
425
|
+
`),"utf8"),C(_(g,"SUMMARY.md"),String(a.answer||""),"utf8"),C(_(g,"CLAIMS.md"),["# Key claims","",...Array.isArray(a.claims)&&a.claims.length?a.claims.map((w)=>{let E=Array.isArray(w.sourceIds)?w.sourceIds.join(", "):"";return`- ${w.claim||""} (${w.support||"support unknown"}${E?`; ${E}`:""})`}):["No structured claims were extracted."],""].join(`
|
|
426
|
+
`),"utf8"),C(_(g,"EVIDENCE.md"),["# Extracted evidence","",...i.length?i.map((w)=>[`## ${w.sourceId||w.url||"Source"}`,w.url?`<${w.url}>`:"",w.rational?`**Rational:** ${w.rational}`:"",w.evidence?`**Evidence:** ${w.evidence}`:"",w.summary?`**Summary:** ${w.summary}`:"",""].filter(Boolean).join(`
|
|
427
|
+
`)):["No goal-based evidence was extracted."],""].join(`
|
|
428
|
+
`),"utf8"),C(_(g,"GAPS.md"),["# Gaps and caveats","","## Caveats",Ke(a.caveats||[]),"","## Research gaps",Ke(b),""].join(`
|
|
429
|
+
`),"utf8"),C(_(y,"manifest.json"),JSON.stringify({...l,floor:s,citationAudit:o},null,2),"utf8"),C(_(y,"rounds.json"),JSON.stringify(e,null,2),"utf8"),C(_(y,"sources.json"),JSON.stringify(r,null,2),"utf8"),C(_(y,"questions.json"),JSON.stringify(u,null,2),"utf8"),C(_(y,"evidence.json"),JSON.stringify(i,null,2),"utf8"),C(_(h,"index.md"),["# Source index","",...v.map((w)=>{let E=w.title||w.url,A=w.finalUrl||w.url,S=w.contentPath?` — ${w.contentPath}`:"";return`- ${w.id||"?"}: [${E}](${A})${S}`}),""].join(`
|
|
430
|
+
`),"utf8");try{Yo(m,{query:t,rounds:e,sources:r,fetchedSources:n,citationAudit:o,citationUrls:d,floor:s,manifest:l})}catch(w){process.stderr.write(`[greedysearch] Provenance sidecar write failed (non-critical): ${w.message}
|
|
431
|
+
`)}return{dir:m,statusPath:_(m,"STATUS.md"),summaryPath:_(g,"SUMMARY.md"),manifestPath:_(y,"manifest.json"),provenancePath:_(m,"provenance.md"),sourceCount:v.length,sourceFiles:v}}function Wo(t){let e="",r=!1,n=!1;for(let i of String(t)){if(n){e+=i,n=!1;continue}if(i==="\\"){e+=i,n=!0;continue}if(i==='"'){r=!r,e+=i;continue}if(r&&i===`
|
|
432
|
+
`)e+="\\n";else if(r&&i==="\r")e+="\\r";else if(r&&i==="\t")e+="\\t";else e+=i}return e}function Pe(t){if(!t)return null;let e=String(t).trim(),r=e.indexOf("BEGIN_JSON"),n=e.indexOf("END_JSON");if(r!==-1&&n!==-1&&r<n)e=e.slice(r+10,n).trim();else{let s=e.indexOf("{");if(s>0)e=e.slice(s)}let i=[e,e.replace(/^```json\s*/i,"").replace(/^```\s*/i,"").replace(/```$/i,"").trim()],a=e.indexOf("{"),o=e.lastIndexOf("}");if(a!==-1&&o!==-1&&a<o)i.push(e.slice(a,o+1));for(let s of[...i]){let l=Wo(s);if(l!==s)i.push(l)}for(let s of i)try{return JSON.parse(s)}catch{}return null}import{mkdirSync as Bo,writeFileSync as Qo}from"node:fs";import{join as Yr}from"node:path";var Vo=Yr(process.cwd(),".dm","greedysearch-sources");function wt(t,e=Vo){return Bo(e,{recursive:!0}),t.map((r)=>{if(!r.content||r.content.length<10)return r;let n=String(r.id||"unknown").replace(/[^a-zA-Z0-9_-]/g,""),i=(r.canonicalUrl||r.url||"").replace(/^https?:\/\//,"").replace(/[^a-zA-Z0-9]/g,"-").slice(0,40),a=`${n}-${i}.md`,o=Yr(e,a),s=`---
|
|
433
|
+
url: ${r.finalUrl||r.url}
|
|
434
|
+
title: ${r.title||""}
|
|
435
|
+
source: ${r.source||"unknown"}
|
|
436
|
+
status: ${r.status||""}
|
|
437
|
+
chars: ${r.contentChars||r.content.length}
|
|
438
|
+
---
|
|
439
|
+
|
|
440
|
+
`;Qo(o,s+r.content,"utf8");let{content:l,...c}=r;return{...c,contentPath:o,contentChars:r.contentChars||l.length}})}import{createRequire as Ko}from"node:module";import{createRequire as zo}from"node:module";import{spawn as bs}from"node:child_process";import{basename as vs}from"node:path";import{dirname as $s,join as _s}from"node:path";import{fileURLToPath as Es}from"node:url";import{existsSync as je,mkdirSync as Ss,readFileSync as dn,writeFileSync as Rs}from"node:fs";import{homedir as Ds}from"node:os";import{join as fn}from"node:path";import{tmpdir as As}from"node:os";import{existsSync as Te,mkdirSync as Cs,readFileSync as pn,writeFileSync as ks}from"node:fs";import{homedir as Os}from"node:os";import{join as gn}from"node:path";import{tmpdir as Ns}from"node:os";import{existsSync as Le,mkdirSync as Qs,readFileSync as bn,writeFileSync as Vs}from"node:fs";import{homedir as Ks}from"node:os";import{join as vn}from"node:path";import{tmpdir as zs}from"node:os";var du=Ko(import.meta.url),pu=zo(import.meta.url),Ie=null;async function Zo(){if(Ie)return Ie;let[{Readability:t},{JSDOM:e},{default:r}]=await Promise.all([import("@mozilla/readability"),import("jsdom"),import("turndown")]),n=new r({headingStyle:"atx",bulletListMarker:"-",codeBlockStyle:"fenced"});return n.addRule("removeDataUrls",{filter:(i)=>i.tagName==="IMG"&&i.getAttribute("src")?.startsWith("data:"),replacement:()=>""}),Ie={Readability:t,JSDOM:e,turndown:n},Ie}var rn="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",nn={"user-agent":rn,accept:"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8","accept-language":"en-US,en;q=0.9","accept-encoding":"gzip, deflate, br","cache-control":"no-cache",pragma:"no-cache","sec-ch-ua":'"Chromium";v="122", "Not(A:Brand";v="24", "Google Chrome";v="122"',"sec-ch-ua-mobile":"?0","sec-ch-ua-platform":'"Windows"',"sec-fetch-dest":"document","sec-fetch-mode":"navigate","sec-fetch-site":"none","sec-fetch-user":"?1","upgrade-insecure-requests":"1"},an=[/^localhost$/i,/^127\.\d+\.\d+\.\d+$/,/^0\.0\.0\.0$/,/^\[::1\]$/,/^10\./,/^172\.(1[6-9]|2\d|3[01])\./,/^192\.168\./,/^169\.254\./,/^fc00:/i,/^fe80:/i,/\.local$/i,/\.internal$/i,/\.localhost$/i],Xo=/^::ffff:(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/i,es=/^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/i,ts=/^(?:0x[0-9a-f]+|0[0-7]*|[1-9][0-9]*)$/i;function rs(t){if(!t||t.includes(":"))return null;let e=t.split(".");if(e.length<1||e.length>4)return null;if(!e.every((i)=>ts.test(i)))return null;let r=e.map((i)=>{if(/^0x/i.test(i))return parseInt(i,16);if(/^0[0-7]+$/.test(i))return parseInt(i,8);return parseInt(i,10)});if(r.some((i)=>!Number.isFinite(i)||i<0))return null;let n;if(r.length===1){if(r[0]>4294967295)return null;n=[r[0]>>>24&255,r[0]>>>16&255,r[0]>>>8&255,r[0]&255]}else if(r.length===2){if(r[0]>255||r[1]>16777215)return null;n=[r[0],r[1]>>>16&255,r[1]>>>8&255,r[1]&255]}else if(r.length===3){if(r[0]>255||r[1]>255||r[2]>65535)return null;n=[r[0],r[1],r[2]>>>8&255,r[2]&255]}else{if(r.some((i)=>i>255))return null;n=r}return n.join(".")}function ns(t){let e=t.match(Xo);if(e)return`${e[1]}.${e[2]}.${e[3]}.${e[4]}`;let r=t.match(es);if(r){let n=parseInt(r[1],16),i=parseInt(r[2],16);if(n>65535||i>65535)return null;return[n>>>8&255,n&255,i>>>8&255,i&255].join(".")}return null}function Wr(t){return an.some((e)=>e.test(t))}function is(t={}){return{...nn,...t}}function _t(t){try{if(typeof t!=="string"||!t.trim())return{blocked:!0,reason:"URL must be a non-empty string"};let e=new URL(t);if(e.protocol!=="http:"&&e.protocol!=="https:")return{blocked:!0,reason:`Protocol not allowed: ${e.protocol}`};let r=e.hostname.toLowerCase();for(let i of an)if(i.test(r))return{blocked:!0,reason:`Private/internal address: ${r}`};if(r.startsWith("[")&&r.endsWith("]")){let i=r.slice(1,-1),a=ns(i);if(a&&Wr(a))return{blocked:!0,reason:`Private/internal address: ${r} (maps to ${a})`}}let n=rs(r);if(n&&Wr(n))return{blocked:!0,reason:`Private/internal address: ${r} (normalizes to ${n})`};return{blocked:!1}}catch(e){return{blocked:!0,reason:`Invalid URL: ${e.message}`}}}function as(t){try{let e=new URL(t);if(!(e.hostname==="github.com"||e.hostname.endsWith(".github.com")))return t;let r=e.pathname.split("/").filter(Boolean);if(r.length<5)return t;let[n,i,a,o,...s]=r;if(a!=="blob")return t;let l=s.join("/");return`https://raw.githubusercontent.com/${n}/${i}/${o}/${l}`}catch{return t}}async function os(t,e={}){let r=_t(t);if(r.blocked)return{ok:!1,url:t,finalUrl:t,status:403,error:`Blocked: ${r.reason}`,needsBrowser:!1};let n=t;if(t=as(t),t!==n)console.error(`[fetcher] Rewrote GitHub URL: ${n.slice(0,60)}... → raw.githubusercontent.com`);let{timeoutMs:i=15000,userAgent:a,signal:o}=e,s=new AbortController,l=setTimeout(()=>s.abort(),i);if(o)o.addEventListener("abort",()=>s.abort(),{once:!0});try{let c=await fetch(t,{method:"GET",headers:{...nn,"user-agent":a||rn},redirect:"follow",signal:s.signal});clearTimeout(l);let u=c.headers.get("content-type")||"",d=c.url,p=c.headers.get("last-modified")||"",f=_t(d);if(f.blocked)return{ok:!1,url:t,finalUrl:d,status:c.status,error:`Blocked: ${f.reason}`,needsBrowser:!1};let m=!1;try{m=new URL(d).hostname.toLowerCase()==="raw.githubusercontent.com"}catch{}if(u.includes("text/plain")&&m){let b=await c.text();return{ok:!0,url:n,finalUrl:d,status:c.status,title:d.split("/").pop()||"GitHub File",byline:"",siteName:"GitHub",lang:"",publishedTime:p,lastModified:p,markdown:b,contentLength:b.length,excerpt:b.slice(0,300).replaceAll(/\n/g," "),needsBrowser:!1}}if(!u.includes("text/html")&&!u.includes("application/xhtml"))return{ok:!1,url:t,finalUrl:d,status:c.status,error:`Unsupported content type: ${u}`,needsBrowser:!1};let g=await c.text(),h=on(c.status,g,d,t);if(h.blocked)return{ok:!1,url:t,finalUrl:d,status:c.status,error:`Blocked: ${h.reason}`,needsBrowser:!0};let y=await sn(g,d),v=ln(y);if(!v.ok)return{ok:!1,url:t,finalUrl:d,status:c.status,error:`Low quality content: ${v.reason}`,needsBrowser:!0};return{ok:!0,url:t,finalUrl:d,status:c.status,title:y.title,byline:y.byline,siteName:y.siteName,lang:y.lang,publishedTime:y.publishedTime||p,lastModified:p,markdown:y.markdown,excerpt:y.excerpt,contentLength:y.markdown.length,needsBrowser:!1}}catch(c){clearTimeout(l);let u=ds(c);return{ok:!1,url:t,finalUrl:t,status:0,error:c.message,needsBrowser:u}}}function on(t,e,r,n){let i=e.match(/<title[^>]*>([^<]*)<\/title>/i)?.[1]?.toLowerCase()||"",a=e.slice(0,30000).toLowerCase(),o=`${i} ${a}`;if(t===403||t===429||t===503)return{blocked:!0,reason:`HTTP ${t}`};let s=[{pattern:/class=["'][^"']*captcha["']|<div[^>]*id=["']captcha/i,reason:"captcha"},{pattern:/g-recaptcha|data-sitekey|i['"]m not a robot/i,reason:"captcha"},{pattern:/checking your browser.{0,100}please wait|cf-browser-verification/i,reason:"cloudflare challenge"},{pattern:/just a moment.{0,50}security check|ddos protection by cloudflare/i,reason:"cloudflare challenge"},{pattern:/unusual traffic.{0,50}from your computer network/i,reason:"unusual traffic"},{pattern:/bot detected|automated.{0,20}request/i,reason:"bot detection"},{pattern:/enable\s+javascript\s+to\s+view|javascript\s+is\s+required.{0,50}enabled/i,reason:"requires javascript"},{pattern:/access denied|accessdenied/i,reason:"access denied"},{pattern:/protected by anubis|anubis uses a proof-of-work/i,reason:"anubis challenge"}];for(let c of s)if(c.pattern.test(o))return{blocked:!0,reason:c.reason};let l=us(n,r,e);if(l)return{blocked:!0,reason:l};return{blocked:!1}}var ss=["accounts.google.com","login.microsoftonline.com","login.live.com","auth0.com","okta.com","auth.mozilla.auth0.com","id.atlassian.com"],ls=["login.","signin.","auth.","sso.","accounts.","idp."],cs=["sign in to continue","log in to continue","authentication required","create an account to continue","subscribe to continue reading","members only"];function us(t,e,r){try{let n=new URL(t),i=new URL(e);if(n.hostname.toLowerCase()===i.hostname.toLowerCase())return;let a=i.hostname.toLowerCase();if(ss.some((s)=>a===s||a.endsWith(`.${s}`)))return`redirected to login (${i.hostname})`;if(ls.some((s)=>a.startsWith(s)))return`redirected to login (${i.hostname})`;let o=r.slice(0,20000).toLowerCase();if(cs.some((s)=>o.includes(s)))return`redirected to login page (${i.hostname})`}catch{}return}function ds(t){let e=t.message.toLowerCase();return e.includes("fetch failed")||e.includes("unable to verify")||e.includes("certificate")||e.includes("timeout")}function Br(t){let e=['meta[property="article:published_time"]','meta[name="article:published_time"]','meta[property="og:published_time"]','meta[name="publication_date"]','meta[name="date"]','meta[itemprop="datePublished"]','time[itemprop="datePublished"]','meta[name="DC.date"]'];for(let r of e){let n=t.querySelector(r),i=n?.getAttribute("content")||n?.getAttribute("datetime")||"";if(i)return i}return""}async function sn(t,e){let{Readability:r,JSDOM:n,turndown:i}=await Zo(),a=new n(t,{url:e});try{let o=a.window.document,s=new r(o).parse();if(s&&s.content){let c=i.turndown(s.content).replaceAll(/\n{3,}/g,`
|
|
441
|
+
|
|
442
|
+
`).trim(),u=s.publishedTime||Br(o)||"";return{title:s.title||o.title||e,byline:s.byline||"",siteName:s.siteName||"",lang:s.lang||"",publishedTime:u,markdown:c,excerpt:c.slice(0,300).replaceAll(/\n/g," ")}}let l=o.body;if(l){let c=l.cloneNode(!0);c.querySelectorAll("script, style, nav, footer, header, aside").forEach((d)=>d.remove());let u=(c.textContent||"").replaceAll(/\s+/g," ").trim();return{title:o.title||e,byline:"",siteName:"",lang:"",publishedTime:Br(o),markdown:u,excerpt:u.slice(0,300)}}return{title:e,byline:"",siteName:"",lang:"",publishedTime:"",markdown:"",excerpt:""}}finally{a.window.close()}}function ln(t){let e=t.markdown.trim().toLowerCase(),r=(t.title||"").toLowerCase();if(t.markdown.trim().length<100)return{ok:!1,reason:"content too short (< 100 chars)"};let n=e.toLowerCase(),i=[{check:()=>n.includes("loading")&&n.includes("please wait"),desc:"loading page"},{check:()=>n.includes("please ensure javascript is enabled"),desc:"requires javascript"},{check:()=>n.includes("enable javascript to view"),desc:"requires javascript"},{check:()=>n.includes("just a moment"),desc:"cloudflare challenge detected in content"},{check:()=>n.includes("verify you are human"),desc:"human verification"},{check:()=>n.includes("captcha required"),desc:"captcha in extracted content"},{check:()=>n.includes("access denied"),desc:"access denied in content"},{check:()=>/^\s{0,10}sign\s{1,5}in\s{0,10}$|^\s{0,10}log\s{1,5}in\s{0,10}$/im.test(e),desc:"login form only"}];for(let{check:a,desc:o}of i)if(a())return{ok:!1,reason:o};if(r.includes("just a moment")||r.includes("checking your browser"))return{ok:!1,reason:"cloudflare challenge page detected in title"};return{ok:!0}}var cn={"user-agent":"GreedySearch/1.0",accept:"application/vnd.github+json","x-github-api-version":"2022-11-28"};function Et(t){try{let e=new URL(t);if(!(e.hostname==="github.com"||e.hostname.endsWith(".github.com")))return null;let r=e.pathname.split("/").filter(Boolean);if(r.length<2)return null;let[n,i]=r;if(r.length===2)return{owner:n,repo:i,type:"root"};if(r.length>=4&&(r[2]==="blob"||r[2]==="tree")){let a=r[2],o=r[3],s=r.slice(4).join("/");return{owner:n,repo:i,type:a,ref:o,path:s}}return null}catch{return null}}async function T(t,e=1e4){let r=new AbortController,n=setTimeout(()=>r.abort(),e);try{let i=await fetch(`https://api.github.com${t}`,{headers:cn,signal:r.signal});if(clearTimeout(n),!i.ok)throw Error(`GitHub API ${i.status}: ${t}`);return await i.json()}catch(i){throw clearTimeout(n),i}}async function fs(t,e){try{let r=await T(`/repos/${t}/${e}/readme`);if(r.content&&r.encoding==="base64")return Buffer.from(r.content,"base64").toString("utf8");return""}catch{return""}}async function Qr(t,e,r="HEAD",n="",i){try{let a;if(r==="HEAD")if(i)a=await T(`/repos/${t}/${e}/git/ref/heads/${i}`).catch(()=>null);else a=await Promise.any([T(`/repos/${t}/${e}/git/ref/heads/main`),T(`/repos/${t}/${e}/git/ref/heads/master`)]).catch(()=>null);else a=await T(`/repos/${t}/${e}/git/ref/heads/${r}`).catch(()=>T(`/repos/${t}/${e}/git/ref/heads/master`).catch(()=>null));if(!a?.object?.sha)return[];let o=(await T(`/repos/${t}/${e}/git/commits/${a.object.sha}`)).tree.sha,s=(await T(`/repos/${t}/${e}/git/trees/${o}`)).tree||[];if(n)s=s.filter((l)=>l.path.startsWith(n));return s.slice(0,50).map((l)=>({path:l.path,type:l.type==="tree"?"dir":"file",size:l.size}))}catch{return[]}}async function ps(t,e,r,n,i=1e4,a){let o=async(l)=>{let c=new AbortController,u=setTimeout(()=>c.abort(),i);try{let d=await fetch(l,{headers:{"user-agent":cn["user-agent"]},signal:c.signal});if(clearTimeout(u),d.ok)return await d.text();throw Error("not ok")}catch{throw clearTimeout(u),Error("failed")}};if(!r||r==="HEAD"){if(a)try{return await o(`https://raw.githubusercontent.com/${t}/${e}/${a}/${n}`)}catch{return null}try{return await Promise.any([o(`https://raw.githubusercontent.com/${t}/${e}/main/${n}`),o(`https://raw.githubusercontent.com/${t}/${e}/master/${n}`)])}catch{return null}}let s=[`https://raw.githubusercontent.com/${t}/${e}/${r}/${n}`,`https://raw.githubusercontent.com/${t}/${e}/master/${n}`];for(let l of s)try{return await o(l)}catch{}return null}async function gs(t){let e=Et(t);if(!e)return{ok:!1,error:"Not a valid GitHub URL"};let{owner:r,repo:n,type:i,ref:a,path:o}=e;try{if(i==="root"||i==="tree"&&!o){let s=await T(`/repos/${r}/${n}`),[l,c]=await Promise.allSettled([fs(r,n),Qr(r,n,a||"HEAD","",s.default_branch)]),u=l.status==="fulfilled"?l.value:"",d=c.status==="fulfilled"?c.value:[],p=s?.description?`
|
|
443
|
+
|
|
444
|
+
> ${s.description}`:"",f=s?.stargazers_count==null?"":` ⭐ ${s.stargazers_count}`,m=s?.language?` · ${s.language}`:"",g=`# ${r}/${n}${f}${m}${p}
|
|
445
|
+
|
|
446
|
+
`;if(u)g+=u.slice(0,6000);else g+=`[No README found]
|
|
447
|
+
|
|
448
|
+
Files:
|
|
449
|
+
${d.map((h)=>` ${h.type==="dir"?"\uD83D\uDCC1":"\uD83D\uDCC4"} ${h.path}`).join(`
|
|
450
|
+
`)}`;return{ok:!0,title:`${r}/${n}`,content:g,tree:d.slice(0,30)}}if(i==="blob"&&o){let s;if(!a||a==="HEAD")try{s=(await T(`/repos/${r}/${n}`)).default_branch}catch{s=void 0}let l=await ps(r,n,a,o,1e4,s);if(l===null)return{ok:!1,error:`File not found: ${o}`};return{ok:!0,title:`${r}/${n}: ${o}`,content:l}}if(i==="tree"&&o){let s;if(!a||a==="HEAD")try{s=(await T(`/repos/${r}/${n}`)).default_branch}catch{s=void 0}let l=await Qr(r,n,a||"HEAD",o,s),c=l.map((u)=>` ${u.type==="dir"?"\uD83D\uDCC1":"\uD83D\uDCC4"} ${u.path}`).join(`
|
|
451
|
+
`);return{ok:!0,title:`${r}/${n}/${o}`,content:`[Directory: ${o}]
|
|
452
|
+
|
|
453
|
+
Files:
|
|
454
|
+
${c}`,tree:l}}return{ok:!1,error:"Unsupported GitHub URL type"}}catch(s){return{ok:!1,error:s.message}}}var hs={"user-agent":"GreedySearch/1.0 (Research Bot)",accept:"application/json"};function ms(t){try{let e=new URL(t),r=e.hostname.toLowerCase();if(!(r==="reddit.com"||r.endsWith(".reddit.com")))return null;let n=e.pathname;if(n.match(/^\/(u|user)\/[^/]+\/?$/i))return{type:"user",cleanUrl:Vr(t)};if(n.match(/^\/r\/[^/]+\/comments\/[^/]+/i))return{type:"post",cleanUrl:Vr(t)};return null}catch{return null}}function Vr(t){try{let e=new URL(t);return`${e.protocol}//${e.hostname}${e.pathname}`}catch{return t}}async function ys(t,e=8000){let r=Date.now();try{let n=t.replace(/\/+$/,"")+".json",i=new AbortController,a=setTimeout(()=>i.abort(),15000),o=await fetch(n,{headers:hs,signal:i.signal});if(clearTimeout(a),!o.ok)throw Error(`Reddit API ${o.status}`);let s=await o.json();if(!Array.isArray(s)||s.length<1)throw Error("Invalid Reddit API response structure");let l=s[0],c=s[1],u=l?.data?.children?.[0]?.data;if(!u)throw Error("No post data in Reddit response");let d=ws(u,c,e);return{ok:!0,url:t,finalUrl:t,status:200,contentType:"text/markdown",lastModified:"",title:u.title||"Reddit Post",byline:`u/${u.author}`,siteName:`r/${u.subreddit}`,lang:"en",publishedTime:new Date(u.created_utc*1000).toISOString(),excerpt:u.selftext?.slice(0,300).replace(/\n/g," ")||"",markdown:d,contentLength:d.length,needsBrowser:!1,duration:Date.now()-r}}catch(n){return{ok:!1,url:t,finalUrl:t,status:0,error:`Reddit fetch failed: ${n.message}`,needsBrowser:!1,duration:Date.now()-r}}}function ws(t,e,r){let n="";if(n+=`# ${t.title}
|
|
455
|
+
|
|
456
|
+
`,n+=`**Subreddit:** r/${t.subreddit} | **Author:** u/${t.author} | **Score:** ${t.score}
|
|
457
|
+
|
|
458
|
+
`,t.selftext)n+=t.selftext,n+=`
|
|
459
|
+
|
|
460
|
+
`;else if(t.url)try{let i=new URL(t.url).hostname.toLowerCase();if(i!=="reddit.com"&&!i.endsWith(".reddit.com"))n+=`**Link:** ${t.url}
|
|
461
|
+
|
|
462
|
+
`}catch{n+=`**Link:** ${t.url}
|
|
463
|
+
|
|
464
|
+
`}if(e?.data?.children?.length>0){n+=`---
|
|
465
|
+
|
|
466
|
+
## Comments
|
|
467
|
+
|
|
468
|
+
`;let i=e.data.children.filter((a)=>a.kind==="t1").slice(0,10);for(let a of i)n+=un(a.data,0),n+=`
|
|
469
|
+
`}if(n.length>r)n=n.slice(0,r).trim()+`
|
|
470
|
+
|
|
471
|
+
... (truncated)`;return n}function un(t,e){if(!t||t.body==="[deleted]"||t.body==="[removed]")return"";let r="> ".repeat(e),n="";if(n+=`${r}**u/${t.author}** (${t.score} pts)
|
|
472
|
+
`,n+=`${r}${t.body.replaceAll(`
|
|
473
|
+
`,`
|
|
474
|
+
`+r)}
|
|
475
|
+
`,e<3&&t.replies?.data?.children){let i=t.replies.data.children.filter((a)=>a.kind==="t1");for(let a of i.slice(0,5))n+=`
|
|
476
|
+
`+un(a.data,e+1)}return n}function ne(t,e=8000){if(!t||t.length<=e)return t;let r=`
|
|
477
|
+
|
|
478
|
+
[...content trimmed...]
|
|
479
|
+
|
|
480
|
+
`,n=e-r.length,i=Math.floor(n*0.75),a=n-i,o=i;while(o>i-100&&t[o]!==`
|
|
481
|
+
`)o--;if(o<=i-100)o=i;let s=t.length-a;while(s<t.length-a+100&&t[s]!==`
|
|
482
|
+
`)s++;if(s>=t.length-a+100)s=t.length-a;let l=t.slice(0,o).trimEnd(),c=t.slice(s).trimStart();return`${l}${r}${c}`}function xs(t=process.env,e=process.execPath){let r=t.GREEDY_SEARCH_NODE||t.NODE_BINARY||t.NODE;if(r?.trim())return r.trim();let n=vs(e||"").toLowerCase();if(n==="node"||n==="node.exe")return e;return"node"}var Ps=$s(Es(import.meta.url)),Is=_s(Ps,"..","bin","cdp.mjs"),js=new Set(["list","snap","eval","shot","html","nav","net","click","clickxy","type","loadall","evalraw","browse","stop","--tab"]);function Ts(t){if(!Array.isArray(t)||t.length===0)throw Error("cdp: args must be a non-empty array");if(t[0]==="test")return t.map((e,r)=>Kr(e,r));if(!js.has(t[0]))throw Error(`cdp: unknown subcommand '${t[0]}'`);return t.map((e,r)=>Kr(e,r))}function Kr(t,e){if(typeof t!=="string")throw Error(`cdp: argv[${e}] must be a string (got ${typeof t})`);if(t.includes("\x00"))throw Error(`cdp: argv[${e}] contains a null byte`);return t}function hn(t,e=30000){return Ls(t,null,e)}function Ls(t,e=null,r=30000){let n=Ts(t);return new Promise((i,a)=>{let o=bs(xs(),[Is,...n],{stdio:[e==null?"ignore":"pipe","pipe","pipe"]});if(e!=null)o.stdin.write(e),o.stdin.end();let s="",l="";o.stdout.on("data",(u)=>s+=u),o.stderr.on("data",(u)=>l+=u);let c=setTimeout(()=>{o.kill(),a(Error(`cdp timeout: ${t[0]}`))},r);o.on("close",(u)=>{if(clearTimeout(c),u===0)i(s.trim());else a(Error(l.trim()||`cdp exit ${u}`))})})}async function zr(t){await hn(["evalraw",t,"Page.addScriptToEvaluateOnNewDocument",JSON.stringify({source:`
|
|
483
|
+
(function() {
|
|
484
|
+
// ── Runtime.enable / CDP detection masking ──────────────
|
|
485
|
+
try { delete window.__REBROWSER_RUNTIME_ENABLE; } catch(_) {}
|
|
486
|
+
try { delete window.__REBROWSER_DEVTOOLS; } catch(_) {}
|
|
487
|
+
try { delete window.__nightmare; } catch(_) {}
|
|
488
|
+
try { delete window.__phantom; } catch(_) {}
|
|
489
|
+
try { delete window.callPhantom; } catch(_) {}
|
|
490
|
+
try { delete window._phantom; } catch(_) {}
|
|
491
|
+
try { delete window.Buffer; } catch(_) {}
|
|
492
|
+
|
|
493
|
+
// Real Chrome without automation should not expose navigator.webdriver at all.
|
|
494
|
+
// A literal false or an own-property getter returning undefined is itself a
|
|
495
|
+
// common stealth tell; remove both instance and prototype properties when the
|
|
496
|
+
// descriptor is configurable (as it is with --disable-blink-features).
|
|
497
|
+
try { delete navigator.webdriver; } catch(_) {}
|
|
498
|
+
try { delete Navigator.prototype.webdriver; } catch(_) {}
|
|
499
|
+
Object.defineProperty(navigator, 'vendor', { get: () => 'Google Inc.', configurable: true });
|
|
500
|
+
Object.defineProperty(navigator, 'platform', { get: () => 'Win32', configurable: true });
|
|
501
|
+
Object.defineProperty(navigator, 'maxTouchPoints', { get: () => 0, configurable: true });
|
|
502
|
+
Object.defineProperty(navigator, 'pdfViewerEnabled', { get: () => true, configurable: true });
|
|
503
|
+
Object.defineProperty(navigator, 'productSub', { get: () => '20030107', configurable: true });
|
|
504
|
+
Object.defineProperty(navigator, 'product', { get: () => 'Gecko', configurable: true });
|
|
505
|
+
var __greedyMimeTypes = null;
|
|
506
|
+
function __makeMimeTypes() {
|
|
507
|
+
var pdf = { type: 'application/pdf', suffixes: 'pdf', description: 'Portable Document Format', enabledPlugin: null };
|
|
508
|
+
var textPdf = { type: 'text/pdf', suffixes: 'pdf', description: 'Portable Document Format', enabledPlugin: null };
|
|
509
|
+
try { Object.setPrototypeOf(pdf, MimeType.prototype); } catch(_) {}
|
|
510
|
+
try { Object.setPrototypeOf(textPdf, MimeType.prototype); } catch(_) {}
|
|
511
|
+
var m = [pdf, textPdf];
|
|
512
|
+
try { Object.setPrototypeOf(m, MimeTypeArray.prototype); } catch(_) {}
|
|
513
|
+
m.item = function item(i) { return this[i] || null; };
|
|
514
|
+
m.namedItem = function namedItem(name) { return Array.prototype.find.call(this, function(x) { return x && x.type === name; }) || null; };
|
|
515
|
+
return m;
|
|
516
|
+
}
|
|
517
|
+
Object.defineProperty(navigator, 'plugins', {
|
|
518
|
+
get: () => {
|
|
519
|
+
__greedyMimeTypes = __greedyMimeTypes || __makeMimeTypes();
|
|
520
|
+
var plugin0 = { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' };
|
|
521
|
+
var plugin1 = { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' };
|
|
522
|
+
var plugin2 = { name: 'Native Client', filename: 'internal-nacl-plugin', description: '' };
|
|
523
|
+
try { Object.setPrototypeOf(plugin0, Plugin.prototype); } catch(_) {}
|
|
524
|
+
try { Object.setPrototypeOf(plugin1, Plugin.prototype); } catch(_) {}
|
|
525
|
+
try { Object.setPrototypeOf(plugin2, Plugin.prototype); } catch(_) {}
|
|
526
|
+
var p = [plugin0, plugin1, plugin2];
|
|
527
|
+
p.item = function item(i) { return this[i] || null; };
|
|
528
|
+
p.namedItem = function namedItem(name) { return Array.prototype.find.call(this, function(x) { return x && x.name === name; }) || null; };
|
|
529
|
+
p.refresh = function refresh() {};
|
|
530
|
+
try { Object.setPrototypeOf(p, PluginArray.prototype); } catch(_) {}
|
|
531
|
+
try {
|
|
532
|
+
__greedyMimeTypes[0].enabledPlugin = p[0];
|
|
533
|
+
__greedyMimeTypes[1].enabledPlugin = p[0];
|
|
534
|
+
} catch(_) {}
|
|
535
|
+
return p;
|
|
536
|
+
},
|
|
537
|
+
configurable: true,
|
|
538
|
+
});
|
|
539
|
+
Object.defineProperty(navigator, 'mimeTypes', {
|
|
540
|
+
get: () => {
|
|
541
|
+
__greedyMimeTypes = __greedyMimeTypes || __makeMimeTypes();
|
|
542
|
+
return __greedyMimeTypes;
|
|
543
|
+
},
|
|
544
|
+
configurable: true,
|
|
545
|
+
});
|
|
546
|
+
Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'], configurable: true });
|
|
547
|
+
try {
|
|
548
|
+
Object.defineProperty(navigator, 'connection', { get: () => ({ effectiveType: '4g', rtt: 50, downlink: 10, downlinkMax: Infinity, saveData: false }), configurable: true });
|
|
549
|
+
} catch(_) {}
|
|
550
|
+
if (!navigator.mediaDevices) {
|
|
551
|
+
Object.defineProperty(navigator, 'mediaDevices', {
|
|
552
|
+
get: () => ({
|
|
553
|
+
enumerateDevices: () => Promise.resolve([
|
|
554
|
+
{ deviceId: 'default', kind: 'audioinput', label: '', groupId: 'default' },
|
|
555
|
+
{ deviceId: 'default', kind: 'audiooutput', label: '', groupId: 'default' },
|
|
556
|
+
{ deviceId: '', kind: 'videoinput', label: '', groupId: '' },
|
|
557
|
+
]),
|
|
558
|
+
getUserMedia: () => Promise.reject(new DOMException('NotAllowedError')),
|
|
559
|
+
getDisplayMedia: () => Promise.reject(new DOMException('NotAllowedError')),
|
|
560
|
+
}),
|
|
561
|
+
configurable: true,
|
|
562
|
+
});
|
|
563
|
+
}
|
|
564
|
+
// ── Missing platform APIs (headless often lacks these) ─
|
|
565
|
+
try {
|
|
566
|
+
if (!navigator.share) {
|
|
567
|
+
navigator.share = function() { return Promise.reject(new Error('NotAllowedError')); };
|
|
568
|
+
}
|
|
569
|
+
} catch(_) {}
|
|
570
|
+
try {
|
|
571
|
+
if (!navigator.contentIndex) {
|
|
572
|
+
Object.defineProperty(navigator, 'contentIndex', { get: () => ({ add: function() {}, delete: function() {}, getAll: function() { return Promise.resolve([]); } }), configurable: true });
|
|
573
|
+
}
|
|
574
|
+
} catch(_) {}
|
|
575
|
+
|
|
576
|
+
if (!window.chrome) {
|
|
577
|
+
window.chrome = {
|
|
578
|
+
app: { isInstalled: false, InstallState: {}, RunningState: {} },
|
|
579
|
+
runtime: {
|
|
580
|
+
OnInstalledReason: {}, OnRestartRequiredReason: {}, PlatformArch: {}, PlatformNaclArch: {}, PlatformOs: {}, RequestUpdateCheckStatus: {},
|
|
581
|
+
connect: () => ({}), sendMessage: () => {}, onMessage: { addListener: () => {} }
|
|
582
|
+
},
|
|
583
|
+
loadTimes: function() { return { requestTime: 0, startLoadTime: Date.now() - 5000, commitLoadTime: Date.now() - 3000, finishDocumentLoadTime: Date.now() - 2000, finishLoadTime: Date.now() - 1000, firstPaintTime: Date.now() - 800, navigationType: 'Other', wasFetchedViaSpdy: true, wasNpnNegotiated: true, npnNegotiatedProtocol: 'h2', wasAlternateProtocolAvailable: false, connectionInfo: 'http/2' }; },
|
|
584
|
+
csi: function() { var t = Date.now(); return { onloadT: t - 2000, startE: t - 5000, pageT: 'back', tran: 2 }; },
|
|
585
|
+
};
|
|
586
|
+
}
|
|
587
|
+
var __greedyNativeFns = [];
|
|
588
|
+
function __markNative(fn) { try { __greedyNativeFns.push(fn); } catch(_) {} return fn; }
|
|
589
|
+
|
|
590
|
+
var origQuery = navigator.permissions?.query;
|
|
591
|
+
if (origQuery) {
|
|
592
|
+
navigator.permissions.query = __markNative(function query(params) {
|
|
593
|
+
if (params && params.name === 'notifications') return Promise.resolve({ state: Notification.permission || 'default', onchange: null });
|
|
594
|
+
return origQuery.apply(this, arguments);
|
|
595
|
+
});
|
|
596
|
+
}
|
|
597
|
+
try {
|
|
598
|
+
var getParam = WebGLRenderingContext.prototype.getParameter;
|
|
599
|
+
WebGLRenderingContext.prototype.getParameter = __markNative(function getParameter(p) {
|
|
600
|
+
if (p === 37445) return 'Intel Inc.';
|
|
601
|
+
if (p === 37446) return 'Intel Iris OpenGL Engine';
|
|
602
|
+
return getParam.call(this, p);
|
|
603
|
+
});
|
|
604
|
+
} catch(_) {}
|
|
605
|
+
// ── WebGL readPixels noise ──────────────────────────
|
|
606
|
+
// CreepJS and other fingerprinters draw content with WebGL and read back the
|
|
607
|
+
// rendered pixels. Adding subtle noise breaks rendering-based fingerprinting.
|
|
608
|
+
try {
|
|
609
|
+
var origReadPixels = WebGLRenderingContext.prototype.readPixels;
|
|
610
|
+
WebGLRenderingContext.prototype.readPixels = __markNative(function readPixels(x, y, width, height, format, type, pixels) {
|
|
611
|
+
var result = origReadPixels.call(this, x, y, width, height, format, type, pixels);
|
|
612
|
+
if (pixels && pixels.length > 0) {
|
|
613
|
+
pixels[0] ^= 1;
|
|
614
|
+
}
|
|
615
|
+
return result;
|
|
616
|
+
});
|
|
617
|
+
} catch(_) {}
|
|
618
|
+
Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8, configurable: true });
|
|
619
|
+
Object.defineProperty(navigator, 'deviceMemory', { get: () => 8, configurable: true });
|
|
620
|
+
|
|
621
|
+
// ── Canvas fingerprint noise ─────────────────────────
|
|
622
|
+
// Headless rendering engines produce slightly different canvas output
|
|
623
|
+
// than headed Chrome. Subtle noise breaks hash-based fingerprinting.
|
|
624
|
+
try {
|
|
625
|
+
var __canvasNoise = ((Date.now() & 0xFF) | 1);
|
|
626
|
+
var origFill = CanvasRenderingContext2D.prototype.fillText;
|
|
627
|
+
CanvasRenderingContext2D.prototype.fillText = __markNative(function fillText() {
|
|
628
|
+
this.globalAlpha = 0.9995;
|
|
629
|
+
return origFill.apply(this, arguments);
|
|
630
|
+
});
|
|
631
|
+
} catch(_) {}
|
|
632
|
+
try {
|
|
633
|
+
var origStroke = CanvasRenderingContext2D.prototype.strokeText;
|
|
634
|
+
CanvasRenderingContext2D.prototype.strokeText = __markNative(function strokeText() {
|
|
635
|
+
this.globalAlpha = 0.9995;
|
|
636
|
+
return origStroke.apply(this, arguments);
|
|
637
|
+
});
|
|
638
|
+
} catch(_) {}
|
|
639
|
+
try {
|
|
640
|
+
var origToDataURL = HTMLCanvasElement.prototype.toDataURL;
|
|
641
|
+
HTMLCanvasElement.prototype.toDataURL = __markNative(function toDataURL() {
|
|
642
|
+
var ctx = this.getContext('2d');
|
|
643
|
+
if (ctx) {
|
|
644
|
+
// Spread noise across canvas to break hash-based fingerprinting.
|
|
645
|
+
// Uses a deterministic pattern so it's consistent per page load
|
|
646
|
+
// but varies between sessions.
|
|
647
|
+
var w = this.width, h = this.height;
|
|
648
|
+
if (w > 0 && h > 0) {
|
|
649
|
+
var imgData = ctx.getImageData(0, 0, Math.min(w, 4), Math.min(h, 4));
|
|
650
|
+
if (imgData && imgData.data) {
|
|
651
|
+
for (var __i = 0; __i < imgData.data.length; __i += 4) {
|
|
652
|
+
imgData.data[__i] ^= (__canvasNoise + __i) & 0xFF;
|
|
653
|
+
}
|
|
654
|
+
ctx.putImageData(imgData, 0, 0);
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
return origToDataURL.apply(this, arguments);
|
|
659
|
+
});
|
|
660
|
+
} catch(_) {}
|
|
661
|
+
|
|
662
|
+
// ── AudioContext fingerprint noise ────────────────────
|
|
663
|
+
// Headless Chrome's AudioContext produces slightly different output.
|
|
664
|
+
// Subtle noise breaks audio-based fingerprinting.
|
|
665
|
+
try {
|
|
666
|
+
var __audioSeed = ((Date.now() & 0x1F) | 1);
|
|
667
|
+
var origGetChannelData = AudioBuffer.prototype.getChannelData;
|
|
668
|
+
AudioBuffer.prototype.getChannelData = __markNative(function getChannelData(channel) {
|
|
669
|
+
var data = origGetChannelData.call(this, channel);
|
|
670
|
+
for (var __i = 0; __i < data.length; __i += 64) {
|
|
671
|
+
data[__i] *= 0.99999;
|
|
672
|
+
}
|
|
673
|
+
return data;
|
|
674
|
+
});
|
|
675
|
+
} catch(_) {}
|
|
676
|
+
|
|
677
|
+
// ── window outer dimensions ──────────────────────────
|
|
678
|
+
// outerWidth/Height = 0 in headless — a well-known bot signal.
|
|
679
|
+
// Mirror innerWidth/Height (set by --window-size flag) so the ratio is sane.
|
|
680
|
+
try {
|
|
681
|
+
if (!window.outerWidth) Object.defineProperty(window, 'outerWidth', { get: () => window.innerWidth || 1920, configurable: true });
|
|
682
|
+
if (!window.outerHeight) Object.defineProperty(window, 'outerHeight', { get: () => window.innerHeight || 1080, configurable: true });
|
|
683
|
+
} catch(_) {}
|
|
684
|
+
|
|
685
|
+
// ── screen properties ─────────────────────────────────
|
|
686
|
+
// Headless Chrome often reports an 800x600 screen even when the viewport is
|
|
687
|
+
// 1920x1080. Keep screen metrics internally consistent with our launch flags.
|
|
688
|
+
try {
|
|
689
|
+
Object.defineProperty(screen, 'width', { get: () => 1920, configurable: true });
|
|
690
|
+
Object.defineProperty(screen, 'height', { get: () => 1080, configurable: true });
|
|
691
|
+
Object.defineProperty(screen, 'availWidth', { get: () => 1920, configurable: true });
|
|
692
|
+
Object.defineProperty(screen, 'availHeight', { get: () => 1040, configurable: true });
|
|
693
|
+
Object.defineProperty(screen, 'colorDepth', { get: () => 24, configurable: true });
|
|
694
|
+
Object.defineProperty(screen, 'pixelDepth', { get: () => 24, configurable: true });
|
|
695
|
+
} catch(_) {}
|
|
696
|
+
|
|
697
|
+
// ── navigator.userAgentData (UA Client Hints) ─────────
|
|
698
|
+
// Derive version from the UA string already set by --user-agent flag so the
|
|
699
|
+
// two APIs are always consistent. Removes any "HeadlessChrome" brand entry.
|
|
700
|
+
try {
|
|
701
|
+
var _uaMajor = (navigator.userAgent.match(new RegExp('Chrome/([0-9]+)')) || [])[1] || '136';
|
|
702
|
+
var _uaFull = (navigator.userAgent.match(new RegExp('Chrome/([0-9.]+)')) || [])[1] || (_uaMajor + '.0.0.0');
|
|
703
|
+
var _brands = [
|
|
704
|
+
{ brand: 'Not)A;Brand', version: '99' },
|
|
705
|
+
{ brand: 'Google Chrome', version: _uaMajor },
|
|
706
|
+
{ brand: 'Chromium', version: _uaMajor },
|
|
707
|
+
];
|
|
708
|
+
Object.defineProperty(navigator, 'userAgentData', {
|
|
709
|
+
get: function() {
|
|
710
|
+
return {
|
|
711
|
+
brands: _brands, mobile: false, platform: 'Windows',
|
|
712
|
+
getHighEntropyValues: function() {
|
|
713
|
+
return Promise.resolve({
|
|
714
|
+
architecture: 'x86', bitness: '64',
|
|
715
|
+
brands: _brands,
|
|
716
|
+
fullVersionList: [
|
|
717
|
+
{ brand: 'Not)A;Brand', version: '99.0.0.0' },
|
|
718
|
+
{ brand: 'Google Chrome', version: _uaFull },
|
|
719
|
+
{ brand: 'Chromium', version: _uaFull },
|
|
720
|
+
],
|
|
721
|
+
mobile: false, model: '', platform: 'Windows',
|
|
722
|
+
platformVersion: '15.0.0', uaFullVersion: _uaFull, wow64: false,
|
|
723
|
+
});
|
|
724
|
+
},
|
|
725
|
+
toJSON: function() { return { brands: _brands, mobile: false, platform: 'Windows' }; },
|
|
726
|
+
};
|
|
727
|
+
},
|
|
728
|
+
configurable: true,
|
|
729
|
+
});
|
|
730
|
+
} catch(_) {}
|
|
731
|
+
|
|
732
|
+
// ── CDP Runtime serialization guard ──────────────────
|
|
733
|
+
// Sites detect CDP by putting a getter on Error.prototype.stack
|
|
734
|
+
// and checking if console.log triggers it (only happens when
|
|
735
|
+
// Runtime domain is enabled). We monkey-patch console methods to
|
|
736
|
+
// strip custom getters from arguments before they reach CDP.
|
|
737
|
+
try {
|
|
738
|
+
var _origLog = console.log, _origError = console.error,
|
|
739
|
+
_origWarn = console.warn, _origDebug = console.debug,
|
|
740
|
+
_origInfo = console.info;
|
|
741
|
+
var _safeArg = function(a) {
|
|
742
|
+
if (a instanceof Error) {
|
|
743
|
+
try { return new Error(a.message); } catch(_) { return a; }
|
|
744
|
+
}
|
|
745
|
+
return a;
|
|
746
|
+
};
|
|
747
|
+
console.log = __markNative(function log() { return _origLog.apply(console, Array.prototype.map.call(arguments, _safeArg)); });
|
|
748
|
+
console.error = __markNative(function error() { return _origError.apply(console, Array.prototype.map.call(arguments, _safeArg)); });
|
|
749
|
+
console.warn = __markNative(function warn() { return _origWarn.apply(console, Array.prototype.map.call(arguments, _safeArg)); });
|
|
750
|
+
console.debug = __markNative(function debug() { return _origDebug.apply(console, Array.prototype.map.call(arguments, _safeArg)); });
|
|
751
|
+
console.info = __markNative(function info() { return _origInfo.apply(console, Array.prototype.map.call(arguments, _safeArg)); });
|
|
752
|
+
} catch(_) {}
|
|
753
|
+
|
|
754
|
+
// ── Native function masking ──────────────────────────
|
|
755
|
+
// Patched APIs should not stringify as user-defined stealth code.
|
|
756
|
+
try {
|
|
757
|
+
var __nativeToString = Function.prototype.toString;
|
|
758
|
+
Function.prototype.toString = function toString() {
|
|
759
|
+
if (__greedyNativeFns.indexOf(this) !== -1) {
|
|
760
|
+
var name = this.name || '';
|
|
761
|
+
return 'function ' + name + '() { [native code] }';
|
|
762
|
+
}
|
|
763
|
+
return __nativeToString.call(this);
|
|
764
|
+
};
|
|
765
|
+
} catch(_) {}
|
|
766
|
+
})();
|
|
767
|
+
`})])}function Ms(t,e=9222){let r=Number.parseInt(String(t??""),10);return Number.isInteger(r)&&r>1024&&r<65535?r:e}var Us=As().replaceAll("\\","/"),Du=Ms(process.env.GREEDY_SEARCH_PORT),G=(process.env.GREEDY_SEARCH_PROFILE_DIR||process.env.CDP_PROFILE_DIR||`${Us}/greedysearch-chrome-profile`).replaceAll("\\","/"),Au=`${G}/DevToolsActivePort`,Cu=process.env.GREEDY_SEARCH_PID_FILE||`${G}/browser.pid`,ku=process.env.CDP_PAGES_CACHE||`${G}/cdp-pages.json`,Ou=process.env.GREEDY_SEARCH_MODE_FILE||`${G}/browser-mode`,Nu=process.env.GREEDY_SEARCH_METADATA_FILE||`${G}/browser-metadata.json`,xu=process.env.GREEDY_SEARCH_LAUNCH_LOCK_FILE||`${G}/browser-launch.lock`,Pu=process.env.GREEDY_SEARCH_ACTIVITY_FILE||`${G}/browser-last-activity`,Iu=(process.env.CDP_SOCKET_DIR||`${G}/cdp-sockets`).replaceAll("\\","/"),ju=`${G}/visible-recovery.jsonl`,St=fn(Ds(),".dm"),Y=fn(St,"greedyconfig"),Rt=["perplexity","google","chatgpt","gemini"],Dt="gemini";function Gs(){try{if(je(Y)){let t=dn(Y,"utf8"),e=JSON.parse(t);if(Array.isArray(e.engines)&&e.engines.length>0&&e.engines.every((r)=>typeof r==="string")){let r=e.engines.filter((i)=>bt[i]),n=e.engines.filter((i)=>!bt[i]);if(n.length>0)process.stderr.write(`[greedysearch] Warning: ignoring unknown engine(s) in ${Y}: ${n.join(", ")}
|
|
768
|
+
[greedysearch] Available engines: ${Object.keys(bt).join(", ")}
|
|
769
|
+
`);if(r.length>0)return r;process.stderr.write(`[greedysearch] Warning: no valid engines in ${Y}, falling back to defaults: ${Rt.join(", ")}
|
|
770
|
+
`)}}}catch{}return Rt}function Fs(){try{if(!je(St))Ss(St,{recursive:!0});if(!je(Y))Rs(Y,JSON.stringify({engines:Rt,synthesizer:Dt},null,2)+`
|
|
771
|
+
`,"utf8")}catch{}}Fs();var Zr=["gemini","chatgpt"];function Hs(){try{if(je(Y)){let t=dn(Y,"utf8"),e=JSON.parse(t);if(typeof e.synthesizer==="string"){let r=e.synthesizer.toLowerCase();if(Zr.includes(r))return r;process.stderr.write(`[greedysearch] Warning: unknown synthesizer "${e.synthesizer}" in ${Y}
|
|
772
|
+
[greedysearch] Available synthesizers: ${Zr.join(", ")}
|
|
773
|
+
[greedysearch] Falling back to default: ${Dt}
|
|
774
|
+
`)}}}catch{}return Dt}var bt={perplexity:"perplexity.mjs",p:"perplexity.mjs",bing:"bing-copilot.mjs",b:"bing-copilot.mjs",google:"google-ai.mjs",g:"google-ai.mjs",gemini:"gemini.mjs",gem:"gemini.mjs",chatgpt:"chatgpt.mjs",gpt:"chatgpt.mjs","semantic-scholar":"semantic-scholar.mjs",semanticscholar:"semantic-scholar.mjs",s2:"semantic-scholar.mjs",logically:"logically.mjs",log:"logically.mjs"},Tu=Gs(),Lu=Hs(),Mu=Math.max(1,Number.parseInt(process.env.GREEDY_FETCH_CONCURRENCY||"5",10)||5);process.env.CDP_PROFILE_DIR=G;function qs(t,e=9222){let r=Number.parseInt(String(t??""),10);return Number.isInteger(r)&&r>1024&&r<65535?r:e}var Js=Ns().replaceAll("\\","/"),Uu=qs(process.env.GREEDY_SEARCH_PORT),F=(process.env.GREEDY_SEARCH_PROFILE_DIR||process.env.CDP_PROFILE_DIR||`${Js}/greedysearch-chrome-profile`).replaceAll("\\","/"),Gu=`${F}/DevToolsActivePort`,Fu=process.env.GREEDY_SEARCH_PID_FILE||`${F}/browser.pid`,Hu=process.env.CDP_PAGES_CACHE||`${F}/cdp-pages.json`,qu=process.env.GREEDY_SEARCH_MODE_FILE||`${F}/browser-mode`,Ju=process.env.GREEDY_SEARCH_METADATA_FILE||`${F}/browser-metadata.json`,Yu=process.env.GREEDY_SEARCH_LAUNCH_LOCK_FILE||`${F}/browser-launch.lock`,Wu=process.env.GREEDY_SEARCH_ACTIVITY_FILE||`${F}/browser-last-activity`,Bu=(process.env.CDP_SOCKET_DIR||`${F}/cdp-sockets`).replaceAll("\\","/"),Qu=`${F}/visible-recovery.jsonl`,At=gn(Os(),".dm"),W=gn(At,"greedyconfig"),Ct=["perplexity","google","chatgpt","gemini"],kt="gemini";function Ys(){try{if(Te(W)){let t=pn(W,"utf8"),e=JSON.parse(t);if(Array.isArray(e.engines)&&e.engines.length>0&&e.engines.every((r)=>typeof r==="string")){let r=e.engines.filter((i)=>vt[i]),n=e.engines.filter((i)=>!vt[i]);if(n.length>0)process.stderr.write(`[greedysearch] Warning: ignoring unknown engine(s) in ${W}: ${n.join(", ")}
|
|
775
|
+
[greedysearch] Available engines: ${Object.keys(vt).join(", ")}
|
|
776
|
+
`);if(r.length>0)return r;process.stderr.write(`[greedysearch] Warning: no valid engines in ${W}, falling back to defaults: ${Ct.join(", ")}
|
|
777
|
+
`)}}}catch{}return Ct}function Ws(){try{if(!Te(At))Cs(At,{recursive:!0});if(!Te(W))ks(W,JSON.stringify({engines:Ct,synthesizer:kt},null,2)+`
|
|
778
|
+
`,"utf8")}catch{}}Ws();var Xr=["gemini","chatgpt"];function Bs(){try{if(Te(W)){let t=pn(W,"utf8"),e=JSON.parse(t);if(typeof e.synthesizer==="string"){let r=e.synthesizer.toLowerCase();if(Xr.includes(r))return r;process.stderr.write(`[greedysearch] Warning: unknown synthesizer "${e.synthesizer}" in ${W}
|
|
779
|
+
[greedysearch] Available synthesizers: ${Xr.join(", ")}
|
|
780
|
+
[greedysearch] Falling back to default: ${kt}
|
|
781
|
+
`)}}}catch{}return kt}var vt={perplexity:"perplexity.mjs",p:"perplexity.mjs",bing:"bing-copilot.mjs",b:"bing-copilot.mjs",google:"google-ai.mjs",g:"google-ai.mjs",gemini:"gemini.mjs",gem:"gemini.mjs",chatgpt:"chatgpt.mjs",gpt:"chatgpt.mjs","semantic-scholar":"semantic-scholar.mjs",semanticscholar:"semantic-scholar.mjs",s2:"semantic-scholar.mjs",logically:"logically.mjs",log:"logically.mjs"},Vu=Ys(),Ku=Bs(),zu=Math.max(1,Number.parseInt(process.env.GREEDY_FETCH_CONCURRENCY||"5",10)||5);process.env.CDP_PROFILE_DIR=F;var Zu=Number.parseInt(process.env.GREEDY_SEARCH_IDLE_TIMEOUT_MINUTES||"5",10)||5,Xu=import.meta.dirname||new URL(".",import.meta.url).pathname.replace(/^\/([A-Z]:)/,"$1"),ed=Number.parseInt(process.env.GREEDY_SEARCH_IDLE_TIMEOUT_MINUTES||"5",10)||5,td=Number.parseInt(process.env.GREEDY_SEARCH_VISIBLE_IDLE_TIMEOUT_MINUTES||"60",10)||60,D=hn;async function mn(){let t=(await D(["list"])).split(`
|
|
782
|
+
`)[0];if(!t)throw Error("No Chrome tabs found");return t.slice(0,8)}async function yn(t="about:blank"){let e=await mn(),r=new URL(t).hostname;if(r==="copilot.microsoft.com"||r==="www.perplexity.ai"||r==="perplexity.ai"||r.endsWith(".perplexity.ai")){let a=await D(["evalraw",e,"Target.createTarget",JSON.stringify({url:"about:blank"})]),{targetId:o}=JSON.parse(a),s=o.slice(0,8);if(await D(["list"]).catch(()=>null),r==="copilot.microsoft.com")await zr(s);else zr(s).catch(()=>{});return await D(["list"]).catch(()=>null),o}let n=await D(["evalraw",e,"Target.createTarget",JSON.stringify({url:t})]),{targetId:i}=JSON.parse(n);return await D(["list"]).catch(()=>null),i}async function wn(t){try{let e=await mn();await D(["evalraw",e,"Target.closeTarget",JSON.stringify({targetId:t})])}catch{}}function Zs(t,e=9222){let r=Number.parseInt(String(t??""),10);return Number.isInteger(r)&&r>1024&&r<65535?r:e}var Xs=zs().replaceAll("\\","/"),od=Zs(process.env.GREEDY_SEARCH_PORT),H=(process.env.GREEDY_SEARCH_PROFILE_DIR||process.env.CDP_PROFILE_DIR||`${Xs}/greedysearch-chrome-profile`).replaceAll("\\","/"),sd=`${H}/DevToolsActivePort`,ld=process.env.GREEDY_SEARCH_PID_FILE||`${H}/browser.pid`,cd=process.env.CDP_PAGES_CACHE||`${H}/cdp-pages.json`,ud=process.env.GREEDY_SEARCH_MODE_FILE||`${H}/browser-mode`,dd=process.env.GREEDY_SEARCH_METADATA_FILE||`${H}/browser-metadata.json`,fd=process.env.GREEDY_SEARCH_LAUNCH_LOCK_FILE||`${H}/browser-launch.lock`,pd=process.env.GREEDY_SEARCH_ACTIVITY_FILE||`${H}/browser-last-activity`,gd=(process.env.CDP_SOCKET_DIR||`${H}/cdp-sockets`).replaceAll("\\","/"),hd=`${H}/visible-recovery.jsonl`,Ot=vn(Ks(),".dm"),B=vn(Ot,"greedyconfig"),Nt=["perplexity","google","chatgpt","gemini"],xt="gemini";function el(){try{if(Le(B)){let t=bn(B,"utf8"),e=JSON.parse(t);if(Array.isArray(e.engines)&&e.engines.length>0&&e.engines.every((r)=>typeof r==="string")){let r=e.engines.filter((i)=>$t[i]),n=e.engines.filter((i)=>!$t[i]);if(n.length>0)process.stderr.write(`[greedysearch] Warning: ignoring unknown engine(s) in ${B}: ${n.join(", ")}
|
|
783
|
+
[greedysearch] Available engines: ${Object.keys($t).join(", ")}
|
|
784
|
+
`);if(r.length>0)return r;process.stderr.write(`[greedysearch] Warning: no valid engines in ${B}, falling back to defaults: ${Nt.join(", ")}
|
|
785
|
+
`)}}}catch{}return Nt}function tl(){try{if(!Le(Ot))Qs(Ot,{recursive:!0});if(!Le(B))Vs(B,JSON.stringify({engines:Nt,synthesizer:xt},null,2)+`
|
|
786
|
+
`,"utf8")}catch{}}tl();var en=["gemini","chatgpt"];function rl(){try{if(Le(B)){let t=bn(B,"utf8"),e=JSON.parse(t);if(typeof e.synthesizer==="string"){let r=e.synthesizer.toLowerCase();if(en.includes(r))return r;process.stderr.write(`[greedysearch] Warning: unknown synthesizer "${e.synthesizer}" in ${B}
|
|
787
|
+
[greedysearch] Available synthesizers: ${en.join(", ")}
|
|
788
|
+
[greedysearch] Falling back to default: ${xt}
|
|
789
|
+
`)}}}catch{}return xt}var $t={perplexity:"perplexity.mjs",p:"perplexity.mjs",bing:"bing-copilot.mjs",b:"bing-copilot.mjs",google:"google-ai.mjs",g:"google-ai.mjs",gemini:"gemini.mjs",gem:"gemini.mjs",chatgpt:"chatgpt.mjs",gpt:"chatgpt.mjs","semantic-scholar":"semantic-scholar.mjs",semanticscholar:"semantic-scholar.mjs",s2:"semantic-scholar.mjs",logically:"logically.mjs",log:"logically.mjs"},md=el(),yd=rl(),tn=Math.max(1,Number.parseInt(process.env.GREEDY_FETCH_CONCURRENCY||"5",10)||5);process.env.CDP_PROFILE_DIR=H;function nl(){if(typeof globalThis.DOMMatrix>"u")globalThis.DOMMatrix=class{constructor(t=void 0){}multiplySelf(){return this}preMultiplySelf(){return this}translateSelf(){return this}scaleSelf(){return this}rotateSelf(){return this}};if(typeof globalThis.ImageData>"u")globalThis.ImageData=class{constructor(t=void 0,e=0,r=0){this.data=t,this.width=e,this.height=r}};if(typeof globalThis.Path2D>"u")globalThis.Path2D=class{constructor(t=void 0){}}}async function il(){nl();let t=await import("pdf-parse"),e=t.PDFParse??t.default;if(!e)throw Error("pdf-parse did not export PDFParse");return e}async function al(t,e){try{let r=new(await il())({data:new Uint8Array(t)});await r.load();let n=await r.getText(),i=n.text?.trim();if(!i)return null;return{title:new URL(e).pathname.split("/").pop()||"Document.pdf",content:`## PDF Content (${n.total} pages)
|
|
790
|
+
|
|
791
|
+
${i}`,pages:n.total}}catch(r){return{error:r.message||String(r)}}}function $n(t="",e=240){let r=String(t).replaceAll(/\s+/g," ").trim();if(r.length<=e)return r;let n=r.slice(0,e),i=n.lastIndexOf(" ");return i>0?`${n.slice(0,i)}...`:`${n}...`}async function ol(t,e,r=8000){let n=Date.now();try{let i=(await D(["evalraw",t,"Page.getFrameTree","{}"]).then((p)=>JSON.parse(p)).catch(()=>null))?.frameTree?.frame?.id||void 0,a=await D(["evalraw",t,"Network.loadNetworkResource",JSON.stringify({frameId:i,url:e,options:{disableCache:!0,includeCredentials:!1}})],20000),o=JSON.parse(a).resource;if(!o?.success||!o.httpStatusCode)return{url:e,error:o?.netErrorName||o?.netError||"loadNetworkResource failed",source:"chrome",duration:Date.now()-n,needsFallback:!0};let s="";if(o.stream)try{let p=await D(["evalraw",t,"IO.read",JSON.stringify({handle:o.stream})],1e4);s=JSON.parse(p).data||"",await D(["evalraw",t,"IO.close",JSON.stringify({handle:o.stream})]).catch(()=>{})}catch{}if(!s||s.length<100)return{url:e,error:"Empty response body from Network.loadNetworkResource",source:"chrome",duration:Date.now()-n,needsFallback:!0};let l=on(o.httpStatusCode,s,e,e);if(l.blocked)return{url:e,status:o.httpStatusCode,error:`Blocked: ${l.reason}`,source:"chrome",duration:Date.now()-n,needsBrowser:!0};let c=await sn(s,e),u=ln(c);if(!u.ok)return{url:e,status:o.httpStatusCode,error:`Low quality: ${u.reason}`,source:"chrome",duration:Date.now()-n,needsBrowser:!0};let d=ne(c.markdown,r);return{url:e,finalUrl:e,status:o.httpStatusCode,contentType:"text/markdown",lastModified:"",publishedTime:c.publishedTime||"",byline:c.byline||"",siteName:c.siteName||"",lang:c.lang||"",title:c.title||e,snippet:c.excerpt,content:d,contentChars:d.length,source:"chrome",duration:Date.now()-n}}catch(i){return{url:e,error:i.message,source:"chrome",duration:Date.now()-n,needsFallback:!0}}}function _n(t){try{return new URL(t).pathname.toLowerCase().endsWith(".pdf")}catch{return!1}}async function sl(t,e=8000){let r=_t(t);if(r.blocked)return{url:t,finalUrl:t,status:403,error:`Blocked: ${r.reason}`,source:"pdf-http"};let n=new AbortController,i=setTimeout(()=>n.abort(),20000),a=Date.now();try{let o=await fetch(t,{method:"GET",redirect:"follow",signal:n.signal,headers:is({accept:"application/pdf,application/octet-stream;q=0.9,*/*;q=0.5"})});clearTimeout(i);let s=o.headers.get("content-type")||"",l=o.url||t,c=Number.parseInt(o.headers.get("content-length")||"0",10);if(o.status>=400)return{url:t,finalUrl:l,status:o.status,error:`HTTP ${o.status}`,source:"pdf-http",duration:Date.now()-a};if(!s.toLowerCase().includes("application/pdf")&&!_n(l))return null;if(c>31457280)return{url:t,finalUrl:l,status:o.status,error:`PDF too large: ${c} bytes`,source:"pdf-http",duration:Date.now()-a};let u=Buffer.from(await o.arrayBuffer()),d=await al(u,l);if(!d||d.error)return{url:t,finalUrl:l,status:o.status,error:d?.error||"PDF text extraction failed",source:"pdf-http",duration:Date.now()-a};let p=ne(d.content,e);return{url:t,finalUrl:l,status:o.status,contentType:"application/pdf",lastModified:o.headers.get("last-modified")||"",title:d.title,snippet:$n(p,320),content:p,contentChars:p.length,pages:d.pages,source:"pdf-http",duration:Date.now()-a}}catch(o){return clearTimeout(i),{url:t,finalUrl:t,error:o.message||String(o),source:"pdf-http",duration:Date.now()-a}}}async function ll(t,e=8000){let r=Date.now();if(_n(t)){let i=await sl(t,e);if(i?.content||i?.status===403)return i}if(Et(t)){let i=Et(t);if(i&&(i.type==="root"||i.type==="tree"||i.type==="blob"&&!i.path?.includes("."))){let a=await gs(t);if(a.ok){let o=ne(a.content,e);return{url:t,finalUrl:t,status:200,contentType:"text/markdown",lastModified:"",title:a.title,snippet:o.slice(0,320),content:o,contentChars:o.length,source:"github-api",...a.tree&&{tree:a.tree},duration:Date.now()-r}}process.stderr.write(`[greedysearch] GitHub API fetch failed, trying HTTP: ${a.error}
|
|
792
|
+
`)}}if(ms(t)?.type==="post"){process.stderr.write(`[greedysearch] Using Reddit JSON API for: ${t.slice(0,60)}...
|
|
793
|
+
`);let i=await ys(t,e);if(i.ok){let a=ne(i.markdown,e);return{url:t,finalUrl:i.finalUrl,status:i.status,contentType:"text/markdown",lastModified:i.lastModified||"",publishedTime:i.publishedTime||"",byline:i.byline||"",siteName:i.siteName||"",lang:i.lang||"",title:i.title,snippet:i.excerpt,content:a,contentChars:a.length,source:"reddit-api",duration:Date.now()-r}}process.stderr.write(`[greedysearch] Reddit API fetch failed, falling back to HTTP: ${i.error}
|
|
794
|
+
`)}let n=await os(t,{timeoutMs:1e4});if(n.ok){let i=ne(n.markdown,e);return{url:t,finalUrl:n.finalUrl,status:n.status,contentType:"text/markdown",lastModified:n.lastModified||"",publishedTime:n.publishedTime||"",byline:n.byline||"",siteName:n.siteName||"",lang:n.lang||"",title:n.title,snippet:n.excerpt,content:i,contentChars:i.length,source:"http",duration:Date.now()-r}}if(n.needsBrowser)try{let i=await yn();try{let a=await ol(i,t,e);if(a.content&&a.content.length>100)return a}finally{await wn(i)}}catch{}return process.stderr.write(`[greedysearch] HTTP failed for ${t.slice(0,60)}, trying browser...
|
|
795
|
+
`),await ul(t,e)}async function cl(t,e=4000,r=200){let n=Date.now()+e;while(Date.now()<n){try{if((await D(["eval",t,'document.readyState === "complete" && !!document.body && document.body.innerText.length > 500'])).trim()==="true")return}catch{}await new Promise((i)=>setTimeout(i,r))}}async function ul(t,e=8000){let r=Date.now(),n;try{n=await yn()}catch(i){return{url:t,title:"",content:null,snippet:"",contentChars:0,error:`openNewTab failed: ${i.message}`,source:"browser",duration:Date.now()-r}}try{await D(["nav",n,t],30000),await cl(n);let i=await D(["eval",n,String.raw`
|
|
796
|
+
(function(){
|
|
797
|
+
var el = document.querySelector('article, [role="main"], main, .post-content, .article-body, #content, .content');
|
|
798
|
+
var text = (el || document.body).innerText;
|
|
799
|
+
return JSON.stringify({
|
|
800
|
+
title: document.title,
|
|
801
|
+
content: text.replace(/\s+/g, ' ').trim(),
|
|
802
|
+
url: location.href
|
|
803
|
+
});
|
|
804
|
+
})()
|
|
805
|
+
`]),a=JSON.parse(i),o=ne(a.content,e);return{url:t,finalUrl:a.url||t,status:200,contentType:"text/plain",lastModified:"",title:a.title,snippet:$n(o,320),content:o,contentChars:o.length,source:"browser",duration:Date.now()-r}}catch(i){return{url:t,title:"",content:null,snippet:"",contentChars:0,error:i.message,source:"browser",duration:Date.now()-r}}finally{await wn(n)}}async function En(t,e=5,r=8000,n=tn){let i=t.slice(0,e);if(i.length===0)return[];let a=Math.min(i.length,Math.max(1,Number.parseInt(String(n),10)||tn));process.stderr.write(`[greedysearch] Fetching content from ${i.length} sources via HTTP (concurrency ${a})...
|
|
806
|
+
`);let o=Array(i.length),s=0,l=0;async function c(){while(!0){let f=s++;if(f>=i.length)return;let m=i[f],g=m.canonicalUrl||m.url;process.stderr.write(`[greedysearch] [${f+1}/${i.length}] Fetching: ${g.slice(0,60)}...
|
|
807
|
+
`);let h=await ll(g,r).catch((y)=>({url:g,title:"",content:null,snippet:"",contentChars:0,error:y.message,source:"error",duration:0}));if(o[f]={id:m.id,...h},h.content&&h.content.length>100)process.stderr.write(`[greedysearch] ✓ ${h.source}: ${h.content.length} chars
|
|
808
|
+
`);else if(h.error)process.stderr.write(`[greedysearch] ✗ ${h.error.slice(0,80)}
|
|
809
|
+
`);l+=1,process.stderr.write(`PROGRESS:fetch:${l}/${i.length}
|
|
810
|
+
`)}}await Promise.all(Array.from({length:a},()=>c()));let u=o.filter((f)=>f.content&&f.content.length>100),d=o.filter((f)=>f.source==="http").length,p=o.filter((f)=>f.source==="browser").length;return process.stderr.write(`[greedysearch] Fetched ${u.length}/${o.length} sources (HTTP: ${d}, Browser: ${p})
|
|
811
|
+
`),o}import{spawn as hl}from"node:child_process";import{basename as dl}from"node:path";function Me(t=process.env,e=process.execPath){let r=t.GREEDY_SEARCH_NODE||t.NODE_BINARY||t.NODE;if(r?.trim())return r.trim();let n=dl(e||"").toLowerCase();if(n==="node"||n==="node.exe")return e;return"node"}import{existsSync as Sn}from"node:fs";import{dirname as fl,join as Ue}from"node:path";import{fileURLToPath as pl}from"node:url";function Rn(t){return pl(new URL(".",t)).replace(/^\/([A-Z]:)/,"$1")}function gl(t,e=Sn){let r=new Set;for(let n of t.filter(Boolean)){if(r.has(n))continue;if(r.add(n),e(n))return n}return t.filter(Boolean).at(-1)}function Dn(t,{moduleDir:e,entrypoint:r=process.argv[1],env:n=process.env,exists:i=Sn}={}){let a=r?fl(r):null,o=n.GREEDY_SEARCH_EXTENSION_DIR?.trim()||null;return gl([e?Ue(e,"..","..","extractors",t):null,e?Ue(e,"..","extractors",t):null,o?Ue(o,"extractors",t):null,a?Ue(a,"..","extractors",t):null],i)}var ml=Rn(import.meta.url),yl={gemini:"gemini.mjs",chatgpt:"chatgpt.mjs"};function wl(t="gemini"){let e=String(t||"gemini").toLowerCase();if(e==="gem")return"gemini";if(e==="gpt")return"chatgpt";return e}async function bl(t,e,{tabPrefix:r=null,timeoutMs:n=180000,visible:i=null}={}){let a=wl(t),o=yl[a];if(!o||!ae.includes(a))throw Error(`Unsupported synthesizer "${t}". Supported: ${ae.join(", ")}`);return new Promise((s,l)=>{let c=Dn(o,{moduleDir:ml}),u=r?["--tab",String(r)]:[],d={...process.env,CDP_PROFILE_DIR:k};if(i!==!0)delete d.GREEDY_SEARCH_VISIBLE,delete d.GREEDY_SEARCH_ALWAYS_VISIBLE;else d.GREEDY_SEARCH_VISIBLE="1",d.GREEDY_SEARCH_ALWAYS_VISIBLE="1";let p=hl(Me(),[c,"--stdin",...u],{stdio:["pipe","pipe","pipe"],env:d});p.stdin.write(e),p.stdin.end();let f="",m="";p.stdout.on("data",(h)=>f+=h),p.stderr.on("data",(h)=>m+=h);let g=setTimeout(()=>{p.kill(),l(Error(`${a} prompt timed out after ${n/1000}s`))},n);p.on("close",(h)=>{if(clearTimeout(g),h!==0){l(Error(m.trim()||`${a} extractor failed`));return}try{s(JSON.parse(f.trim()))}catch{l(Error(`bad JSON from ${a}: ${f.slice(0,100)}`))}})})}async function Pt(t,e={}){return bl("gemini",t,e)}function vl(t){if(t<1000)return"0s";let e=Math.round(t/1000);if(e<60)return`${e}s`;let r=Math.floor(e/60),n=e%60;return`${r}m ${n}s`}function $l(t,e=20){let r=Math.round(t*e),n=e-r;return"["+"█".repeat(r)+"░".repeat(n)+"]"}function An({totalActions:t=0,totalRounds:e=0,totalFetches:r=0,silent:n=!1}={}){let i=Date.now(),a=0,o=0,s=0,l=[],c=null,u=null,d=0,p=500;function f(y){if(l.push(y),l.length>5)l.shift()}function m(){if(l.length===0)return null;return l.reduce((y,v)=>y+v,0)/l.length}function g(y){let v=Date.now()-i,b=t+r+e,w=a+s+o,E=b>0?Math.min(1,w/b):0,A=$l(E),S=m(),Ge=Math.max(0,b-w),le=S?S*Ge:null,ce=le?vl(le):"—",Z=u?` ${u}`:"";return`${A} ${w}/${b} (${y}${Z}, ETA ${ce})`}function h(y){if(n)return;let v=Date.now();if(v-d<p&&y!=="done")return;d=v,process.stderr.write(`[greedysearch] ${g(y)}
|
|
812
|
+
`)}return{startRound(y){o=y-1},endRound(){o++,h("round")},startAction(y,v){c=Date.now(),u=`${y}:${(v||"").slice(0,40)}`,h(y)},endAction(){if(c)f(Date.now()-c),c=null;a++,h("action")},startFetch(y){c=Date.now(),u=`fetch:${(y||"").slice(0,40)}`,h("fetch")},endFetch(y=!0){if(c)f(Date.now()-c),c=null;s++,h(y?"fetch":"fetch-failed")},print(){h("progress")},finish(){h("done")},getElapsedMs(){return Date.now()-i}}}import{spawn as _l}from"node:child_process";import{join as El}from"node:path";import{fileURLToPath as Sl}from"node:url";var Rl=Sl(new URL(".",import.meta.url)).replace(/^\/([A-Z]:)/,"$1"),Dl=El(Rl,"..","..","bin","search.mjs");function Cn(t,e=1/0){let r=new Set,n=[];for(let i of t||[]){let a=te(String(i||""),1000);if(!a||r.has(a))continue;if(r.add(a),n.push(a),n.length>=e)break}return n}function Al(t){let e=String(t||"").trim();if(!e)return[];return[`${e} — definition and overview`,`${e} — how it works, mechanism, or key details`,`${e} — current usage, comparison, or best practices`]}function Cl(t,e){let r=new Map;for(let n of t||[]){let i=n?.canonicalUrl||n?.finalUrl||n?.url;if(i)r.set(i,n)}for(let n of e||[]){let i=n?.canonicalUrl||n?.finalUrl||n?.url;if(!i)continue;if(r.has(i)){let a={...r.get(i),angles:[...r.get(i).angles||[r.get(i).query||""],n.query||""]};r.set(i,a)}else r.set(i,n)}return Array.from(r.values())}var kl=pe.join("|"),Ol=new RegExp(`^\\[(${kl})\\]`);function Nl(t){return/^PROGRESS:/.test(t)||/^\[greedysearch\]/.test(t)||Ol.test(t)||/^GreedySearch Chrome/.test(t)||/^Launching GreedySearch Chrome/.test(t)||/^Headless mode/.test(t)||/^Ready\.?$/.test(t)}async function xl(t,{locale:e=null,short:r=!0}={}){let n=[Dl,"all","--inline","--stdin","--fast"];if(!r)n.push("--full");if(e)n.push("--locale",e);return new Promise((i,a)=>{let o=_l(Me(),n,{stdio:["pipe","pipe","pipe"],env:{...process.env,GREEDY_SEARCH_RESEARCH_CHILD:"1"}});o.stdin.write(t),o.stdin.end();let s="",l="",c="";o.stdout.on("data",(d)=>s+=d),o.stderr.on("data",(d)=>{l+=d,c+=d.toString();let p=c.split(`
|
|
813
|
+
`);c=p.pop()||"";for(let f of p)if(Nl(f))process.stderr.write(`${f}
|
|
814
|
+
`)});let u=setTimeout(()=>{o.kill(),a(Error(`research child search timed out for: ${t}`))},140000);o.on("close",(d)=>{if(clearTimeout(u),d!==0){a(Error(l.trim()||`search child exited with code ${d}`));return}try{i(JSON.parse(s.trim()))}catch{a(Error(`Invalid JSON from research child: ${s.slice(0,200)}`))}})})}function Pl(t,e){let r=new Map;for(let n of e||[]){let i=he(n?.canonicalUrl||n?.finalUrl||n?.url);if(i&&n?.id)r.set(i,n.id)}return(t||[]).map((n,i)=>{let a=he(n?.finalUrl||n?.canonicalUrl||n?.url);return{...n,id:n?.id||r.get(a)||`F${i+1}`}})}function Il(t){let e=t.length,r=t.filter((n)=>n.status==="closed").length;return{total:e,closed:r,open:Math.max(0,e-r)}}async function Kd({query:t,locale:e=null,maxSources:r=5,qualityThreshold:n=8.5,writeBundle:i=process.env.GREEDY_RESEARCH_BUNDLE!=="0",researchOutDir:a=null}={}){let o=new Date().toISOString(),s=Date.now(),l=Hr(t),c=new Set;process.stderr.write(`[greedysearch] Simple research mode: single-pass for "${te(t,80)}"
|
|
815
|
+
`);let p=An({totalActions:6,totalRounds:1,totalFetches:1,silent:process.env.GREEDY_RESEARCH_QUIET==="1"});p.startRound(1);let f=[],m=[],g=Al(t),h=[];p.startAction("search",`${g.length} angles in parallel`);let y=await Promise.allSettled(g.map(($)=>xl($,{locale:e,short:!0})));for(let $=0;$<g.length;$++){let Q=g[$],ie=y[$];if(p.endAction(),ie.status==="fulfilled"){let de=ie.value;h.push({angle:Q,result:de});let V=Be(de,Q);f=Cl(f,V)}else process.stderr.write(`[greedysearch] Simple search angle "${Q}" failed: ${ie.reason.message}
|
|
816
|
+
`)}if(process.stderr.write(`PROGRESS:research:simple:fetching
|
|
817
|
+
`),f.length>0)try{p.startFetch(`top ${Math.min(r,f.length)} sources`),m=await En(f,Math.min(r,f.length),8000,Math.min(3,r)),p.endFetch(!0),f=Ut(f,m)}catch($){p.endFetch(!1),process.stderr.write(`[greedysearch] Source fetching failed: ${$.message}
|
|
818
|
+
`)}m=Pl(m,f),process.stderr.write(`PROGRESS:research:simple:evidence
|
|
819
|
+
`);let v=[];try{let $=await Tr({query:t,questions:l,fetchedSources:m,extractedSourceKeys:c});v=$.evidence||[];for(let Q of $.evidence){let ie=Array.isArray(Q.answers)?Q.answers:[];for(let V of ie){let ee=V?.id||V?.question;if(ee){let X=l.find((kn)=>kn.id===ee);if(X){if(X.status="closed",X.closedRound=1,V.evidence)X.evidence=Cn([...X.evidence||[],V.evidence],4)}}}let de=Array.isArray(Q.newQuestions)?Q.newQuestions:[];for(let V of de){let ee=te(String(V),320);if(ee&&!l.some((X)=>X.question===ee))l.push({id:`Q${l.length+1}`,question:ee,status:"open",reason:"Discovered gap/follow-up",createdRound:1,evidence:[],sourceIds:[]})}}}catch($){process.stderr.write(`[greedysearch] Evidence extraction failed: ${$.message}
|
|
820
|
+
`)}process.stderr.write(`PROGRESS:research:simple:synthesizing
|
|
821
|
+
`);let b={answer:"",agreement:{level:"mixed",summary:"Single-pass synthesis."},differences:[],caveats:[],claims:[],recommendedSources:f.slice(0,4).map(($)=>$.id),synthesized:!1};if(v.length>0)try{p.startAction("synth-evidence","from evidence");let $=await Pt(Mr(t,f,l,v),{timeoutMs:120000});p.endAction(),b={...b,...Pe($?.answer||"")||{}},b.synthesized=Array.isArray(b.claims)&&b.claims.length>0}catch($){process.stderr.write(`[greedysearch] Evidence synthesis failed: ${$.message}
|
|
822
|
+
`)}if(!b.synthesized&&f.length>0)try{p.startAction("synth-final","fallback report");let $=await Pt(Lr(t,[{round:1,learnings:[],gaps:[],actions:[]}],f,l,v),{timeoutMs:120000});p.endAction(),b={...b,...Pe($?.answer||"")||{}},b.synthesized=Array.isArray(b.claims)&&b.claims.length>0}catch($){process.stderr.write(`[greedysearch] Final synthesis failed: ${$.message}
|
|
823
|
+
`)}process.stderr.write(`PROGRESS:research:simple:audit
|
|
824
|
+
`);let w=Ur(b.answer||"",f),E=await Gr(f,w);qr(l,b,w);let A=Cn(b.caveats||[]),S=Fr({sources:f,fetchedSources:m,synthesis:b,citationAudit:w,gaps:A,questions:l,rounds:[{round:1,actions:[],learnings:[],gaps:A}],qualityScore:b.synthesized?8:5,qualityThreshold:n,maxSources:r}),Ge=new Date().toISOString(),le=Date.now()-s,ce={startedAt:o,finishedAt:Ge,durationMs:le,rounds:1,terminationReason:"simple_single_pass"},Z=null,ue;if(i){process.stderr.write(`PROGRESS:research:simple:bundle
|
|
825
|
+
`);try{Z=await Jr({query:t,rounds:[{round:1,actions:[],learnings:[],gaps:A,evidence:v}],sources:f,fetchedSources:m,evidenceItems:v,synthesis:b,citationAudit:w,citationUrls:E,floor:S,manifest:{...ce,engines:Tt,synthesizer:"gemini",actionsRun:1,searches:1,fetches:m.length,sourcesFetched:m.filter(($)=>$?.contentChars>100).length,engineFailures:[],floorMet:S.floorMet},allGaps:A,questions:l,outDir:a}),ue=Z.sourceFiles,delete Z.sourceFiles}catch($){process.stderr.write(`[greedysearch] Research bundle write failed: ${$.message}
|
|
826
|
+
`),Z={error:$.message||String($)},ue=await wt(m)}}else ue=await wt(m);return process.stderr.write(`PROGRESS:research:done
|
|
827
|
+
`),p.endRound(),p.finish(),{query:t,_research:{mode:"simple",breadth:1,iterations:1,maxSources:r,rounds:[{round:1,actions:[],learnings:[],gaps:A,evidence:v}],learnings:[],gaps:A,evidence:v,questions:l,questionProgress:Il(l),qualityHistory:[b.synthesized?8:5],terminationReason:"simple_single_pass",qualityThreshold:n,floor:S,bundle:Z,manifest:ce},_citationAudit:w,_citationUrls:E,_sources:f,_fetchedSources:ue,_synthesis:b,_confidence:{sourcesCount:f.length,fetchedSourceSuccessRate:m.length>0?m.filter(($)=>$.contentChars>100).length/m.length:0,agreementLevel:b.agreement?.level||"mixed",floorMet:S.floorMet}}}export{Kd as runSimpleResearchMode,Cl as mergeSourcesByUrl,Al as buildSearchAngles};
|