@searchos/bot-proxy 1.0.1 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1 -1
- package/dist/index.js +1 -1
- package/package.json +1 -1
- package/README.md +0 -1
package/dist/index.cjs
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
"use strict";var
|
|
1
|
+
"use strict";var G=Object.create;var R=Object.defineProperty;var q=Object.getOwnPropertyDescriptor;var Q=Object.getOwnPropertyNames;var Y=Object.getPrototypeOf,Z=Object.prototype.hasOwnProperty;var tt=(e,o)=>{for(var t in o)R(e,t,{get:o[t],enumerable:!0})},v=(e,o,t,s)=>{if(o&&typeof o=="object"||typeof o=="function")for(let n of Q(o))!Z.call(e,n)&&n!==t&&R(e,n,{get:()=>o[n],enumerable:!(s=q(o,n))||s.enumerable});return e};var et=(e,o,t)=>(t=e!=null?G(Y(e)):{},v(o||!e||!e.__esModule?R(t,"default",{value:e,enumerable:!0}):t,e)),ot=e=>v(R({},"__esModule",{value:!0}),e);var pt={};tt(pt,{cloudflareProxy:()=>z,nextjsProxy:()=>W});module.exports=ot(pt);var k=["chatgpt-user","claude-user","perplexity-user","gemini-user","gemini-deep-research","mistralai-user","grok-user"],U=["gptbot","oai-searchbot","claudebot","claude-searchbot","google-extended","google-cloudvertexbot","perplexitybot","meta-externalagent","applebot-extended","amazonbot","cohere-ai","bytespider","deepseekbot","grokbot","ccbot","diffbot","ai2bot","petalbot"],X=["googlebot","googlebot-image","googlebot-video","googlebot-news","googleother","mediapartners-google","adsbot-google","google-inspectiontool","bingbot","bingpreview","naverbot","yeti","daumoa","baiduspider","yandexbot","duckduckbot","applebot"],V=["facebookexternalhit","linkedinbot","twitterbot","slackbot","discordbot","telegrambot","whatsapp","pinterestbot","redditbot"],J=["ahrefsbot","semrushbot","mj12bot","dotbot","dataforseobot"],nt=["uptimerobot","lighthouse","chrome-lighthouse"],D=new Set([...k,...U]),A=new Set(k),rt=[...k,...U,...X,...V,...J,...nt];function I(e){let o=e.toLowerCase();for(let t of rt)if(o.includes(t)){let s;return A.has(t)?s="ai-user":D.has(t)?s="ai-crawl":X.includes(t)?s="search":V.includes(t)?s="sns":J.includes(t)?s="seo":s="monitoring",{name:t,isAi:D.has(t),category:s}}return null}async function st(e){if(typeof process<"u"&&process.versions?.node){let{createHash:s}=require("crypto");return s("md5").update(e,"utf8").digest("hex")}let o=new TextEncoder().encode(e),t=await crypto.subtle.digest("MD5",o);return Array.from(new Uint8Array(t),s=>s.toString(16).padStart(2,"0")).join("")}function at(e){if(!e)return"/";if(e.startsWith("http://")||e.startsWith("https://"))try{e=new URL(e).pathname}catch{}let o=e.indexOf("?");return o!==-1&&(e=e.substring(0,o)),e?e.split("/").map(t=>{if(!t)return t;try{return encodeURIComponent(decodeURIComponent(t))}catch{return encodeURIComponent(t)}}).join("/"):"/"}async function w(e,o){let t=at(e);return`${await st(t)}_${o}`}var M=/^\/(?:robots\.txt|llms\.txt|sitemap[^\/]*\.(?:xml|xsl))$/i,it=/mobile|android|iphone|phone|tablet|webos|ipad/i;async function W(e,o){let{projectId:t,cloudfrontDomain:s,prerenderDomain:n,botLogEndpoint:i,aiRender:u=!0,ocr:m=!0,aiPathPrefixes:T=[]}=o,p=o.siteDomain.replace(/^https?:\/\//,"").replace(/\/+$/,""),f=s.replace(/\/$/,""),_=new URL(e.url),r=_.pathname;if(e.headers.get("x-prerender-host")){if(M.test(r)||r===`/${t}.txt`){let{NextResponse:a}=await import("next/server");return a.next()}return null}if(r===`/${t}.txt`)try{let a=await fetch(`${n}${r}`,{redirect:"manual"});return new Response(a.body,{status:a.status,headers:{"Content-Type":"text/plain","Cache-Control":"public, max-age=3600"}})}catch{return null}if(M.test(r))try{let a=await fetch(`${n}/static${r}`,{headers:{"X-Signature":t,"X-Original-Host":p}});return new Response(a.body,{status:a.status,headers:{"Content-Type":a.headers.get("Content-Type")||"text/plain","Cache-Control":"public, max-age=3600"}})}catch{return null}let l=e.headers.get("user-agent")||"",h=I(l);if(!h)return null;let d=it.test(l)?"mobile":"desktop",y=`https://${p}${r}`,x=T.length===0||T.some(a=>r.startsWith(a));try{if(h.isAi&&u&&x){let $=await w(r,"ai"),c=await fetch(`${f}/v2/${p}/${$}.html`,{headers:{Accept:"text/html"}});if(c.ok){let g=await c.text();if(g&&g.length>100)return O(i,t,p,r,_.search,l,"hit_ai"),B(n,t,y,m,T),P(g,"HIT-AI",h.name)}let C=await w(r,"desktop"),E=await fetch(`${f}/v2/${p}/${C}.html`,{headers:{Accept:"text/html"}});if(E.ok){let g=await E.text();if(g&&g.length>100)return O(i,t,p,r,_.search,l,"hit_desktop_fallback"),B(n,t,y,m,T),P(g,"HIT-DESKTOP",h.name)}return O(i,t,p,r,_.search,l,"miss_ai"),B(n,t,y,m,T),j(n,t,y),null}let a=await w(r,d),b=await fetch(`${f}/v2/${p}/${a}.html`,{headers:{Accept:"text/html"}});if(b.ok){let $=await b.text();if($&&$.length>100)return O(i,t,p,r,_.search,l,"hit"),j(n,t,y),P($,"HIT",h.name)}}catch{}return O(i,t,p,r,_.search,l,"miss"),j(n,t,y),null}function P(e,o,t){return new Response(e,{headers:{"Content-Type":"text/html; charset=utf-8","X-Cache":o,"X-Bot":t}})}function O(e,o,t,s,n,i,u){fetch(e,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({project_id:o,url:`https://${t}${s}${n}`,domain:t,path:s,user_agent:i,phase:u})}).catch(()=>{})}function j(e,o,t){fetch(`${e}/v2/cache`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({items:[{url:t,project_id:o}]})}).catch(()=>{})}function B(e,o,t,s,n){let i={items:[{url:t,project_id:o}],ocr:s};n.length>0&&(i.path_prefixes=n),fetch(`${e}/v2/cache/ai`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(i)}).catch(()=>{})}var ct=/^\/(?:robots\.txt|llms\.txt|sitemap[^\/]*\.(?:xml|xsl))$/i,lt=/mobile|android|iphone|phone|tablet|webos|ipad/i;async function z(e,o,t,s){let{projectId:n,siteDomain:i,prerenderDomain:u,botLogEndpoint:m,aiRender:T=!0,ocr:p=!0,aiPathPrefixes:f=[]}=s,r=new URL(e.url).pathname,l=i.replace(/^https?:\/\//,"").replace(/\/+$/,"");if(e.headers.get("x-prerender-host"))return null;if(r===`/${n}.txt`)try{let c=await fetch(`${u}${r}`,{redirect:"manual"});return new Response(c.body,{status:c.status,headers:{"Content-Type":"text/plain","Cache-Control":"public, max-age=3600"}})}catch{return new Response("",{status:404})}if(ct.test(r))try{let c=await fetch(`${u}/static${r}`,{redirect:"manual",headers:{"X-Signature":n,"X-Original-Host":l}});return new Response(c.body,{status:c.status,headers:{"Content-Type":c.headers.get("Content-Type")||"text/plain","Cache-Control":"public, max-age=3600"}})}catch{return null}let h=e.headers.get("user-agent")||"",d=I(h);if(!d||!o.V2_CACHE)return null;let y=lt.test(h)?"mobile":"desktop",x=`https://${l}${r}`,a=A.has(d.name)?"ai-user":d.isAi?"ai-crawl":"standard",b=c=>t.waitUntil(c().catch(C=>console.error("[bot-proxy]",C))),$=f.length===0||f.some(c=>r.startsWith(c));try{if(d.isAi&&T&&$){let E=await w(r,"ai"),g=await o.V2_CACHE.get(E);if(g)return b(async()=>{console.log(`[BOT] ${d.name} | ${a} | hit_ai | ${r}`),await S(m,n,l,r,h,"hit_ai"),await K(u,n,x,p,f)}),N(g,"HIT-AI",d.name);let F=await w(r,"desktop"),H=await o.V2_CACHE.get(F);return H?(b(async()=>{console.log(`[BOT] ${d.name} | ${a} | hit_desktop_fallback | ${r}`),await S(m,n,l,r,h,"hit_desktop_fallback"),await K(u,n,x,p,f)}),N(H,"HIT-DESKTOP",d.name)):(b(async()=>{console.log(`[BOT] ${d.name} | ${a} | miss_ai | ${r}`),await S(m,n,l,r,h,"miss_ai"),await Promise.all([K(u,n,x,p,f),L(u,n,x)])}),null)}let c=await w(r,y),C=await o.V2_CACHE.get(c);return C?(b(async()=>{console.log(`[BOT] ${d.name} | ${a} | hit | ${r}`),await S(m,n,l,r,h,"hit"),await L(u,n,x)}),N(C,"HIT",d.name)):(b(async()=>{console.log(`[BOT] ${d.name} | ${a} | miss | ${r}`),await S(m,n,l,r,h,"miss"),await L(u,n,x)}),null)}catch(c){return console.error("[bot-proxy]",c),null}}function N(e,o,t){return new Response(e,{headers:{"Content-Type":"text/html; charset=utf-8","X-Cache":o,"X-Bot":t}})}async function S(e,o,t,s,n,i){await fetch(e,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({project_id:o,url:`https://${t}${s}`,domain:t,path:s,user_agent:n,phase:i})})}async function L(e,o,t){await fetch(`${e}/v2/cache/kv`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({items:[{url:t,project_id:o}]})})}async function K(e,o,t,s,n){let i={items:[{url:t,project_id:o}],ocr:s};n.length>0&&(i.path_prefixes=n),await fetch(`${e}/v2/cache/kv/ai`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(i)})}0&&(module.exports={cloudflareProxy,nextjsProxy});
|
package/dist/index.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
var
|
|
1
|
+
var M=(e=>typeof require<"u"?require:typeof Proxy<"u"?new Proxy(e,{get:(n,t)=>(typeof require<"u"?require:n)[t]}):e)(function(e){if(typeof require<"u")return require.apply(this,arguments);throw Error('Dynamic require of "'+e+'" is not supported')});var I=["chatgpt-user","claude-user","perplexity-user","gemini-user","gemini-deep-research","mistralai-user","grok-user"],v=["gptbot","oai-searchbot","claudebot","claude-searchbot","google-extended","google-cloudvertexbot","perplexitybot","meta-externalagent","applebot-extended","amazonbot","cohere-ai","bytespider","deepseekbot","grokbot","ccbot","diffbot","ai2bot","petalbot"],D=["googlebot","googlebot-image","googlebot-video","googlebot-news","googleother","mediapartners-google","adsbot-google","google-inspectiontool","bingbot","bingpreview","naverbot","yeti","daumoa","baiduspider","yandexbot","duckduckbot","applebot"],U=["facebookexternalhit","linkedinbot","twitterbot","slackbot","discordbot","telegrambot","whatsapp","pinterestbot","redditbot"],X=["ahrefsbot","semrushbot","mj12bot","dotbot","dataforseobot"],W=["uptimerobot","lighthouse","chrome-lighthouse"],H=new Set([...I,...v]),k=new Set(I),z=[...I,...v,...D,...U,...X,...W];function R(e){let n=e.toLowerCase();for(let t of z)if(n.includes(t)){let s;return k.has(t)?s="ai-user":H.has(t)?s="ai-crawl":D.includes(t)?s="search":U.includes(t)?s="sns":X.includes(t)?s="seo":s="monitoring",{name:t,isAi:H.has(t),category:s}}return null}async function F(e){if(typeof process<"u"&&process.versions?.node){let{createHash:s}=M("crypto");return s("md5").update(e,"utf8").digest("hex")}let n=new TextEncoder().encode(e),t=await crypto.subtle.digest("MD5",n);return Array.from(new Uint8Array(t),s=>s.toString(16).padStart(2,"0")).join("")}function G(e){if(!e)return"/";if(e.startsWith("http://")||e.startsWith("https://"))try{e=new URL(e).pathname}catch{}let n=e.indexOf("?");return n!==-1&&(e=e.substring(0,n)),e?e.split("/").map(t=>{if(!t)return t;try{return encodeURIComponent(decodeURIComponent(t))}catch{return encodeURIComponent(t)}}).join("/"):"/"}async function w(e,n){let t=G(e);return`${await F(t)}_${n}`}var V=/^\/(?:robots\.txt|llms\.txt|sitemap[^\/]*\.(?:xml|xsl))$/i,q=/mobile|android|iphone|phone|tablet|webos|ipad/i;async function Q(e,n){let{projectId:t,cloudfrontDomain:s,prerenderDomain:r,botLogEndpoint:i,aiRender:u=!0,ocr:m=!0,aiPathPrefixes:T=[]}=n,p=n.siteDomain.replace(/^https?:\/\//,"").replace(/\/+$/,""),f=s.replace(/\/$/,""),_=new URL(e.url),o=_.pathname;if(e.headers.get("x-prerender-host")){if(V.test(o)||o===`/${t}.txt`){let{NextResponse:a}=await import("next/server");return a.next()}return null}if(o===`/${t}.txt`)try{let a=await fetch(`${r}${o}`,{redirect:"manual"});return new Response(a.body,{status:a.status,headers:{"Content-Type":"text/plain","Cache-Control":"public, max-age=3600"}})}catch{return null}if(V.test(o))try{let a=await fetch(`${r}/static${o}`,{headers:{"X-Signature":t,"X-Original-Host":p}});return new Response(a.body,{status:a.status,headers:{"Content-Type":a.headers.get("Content-Type")||"text/plain","Cache-Control":"public, max-age=3600"}})}catch{return null}let l=e.headers.get("user-agent")||"",h=R(l);if(!h)return null;let d=q.test(l)?"mobile":"desktop",y=`https://${p}${o}`,x=T.length===0||T.some(a=>o.startsWith(a));try{if(h.isAi&&u&&x){let $=await w(o,"ai"),c=await fetch(`${f}/v2/${p}/${$}.html`,{headers:{Accept:"text/html"}});if(c.ok){let g=await c.text();if(g&&g.length>100)return O(i,t,p,o,_.search,l,"hit_ai"),j(r,t,y,m,T),A(g,"HIT-AI",h.name)}let C=await w(o,"desktop"),E=await fetch(`${f}/v2/${p}/${C}.html`,{headers:{Accept:"text/html"}});if(E.ok){let g=await E.text();if(g&&g.length>100)return O(i,t,p,o,_.search,l,"hit_desktop_fallback"),j(r,t,y,m,T),A(g,"HIT-DESKTOP",h.name)}return O(i,t,p,o,_.search,l,"miss_ai"),j(r,t,y,m,T),P(r,t,y),null}let a=await w(o,d),b=await fetch(`${f}/v2/${p}/${a}.html`,{headers:{Accept:"text/html"}});if(b.ok){let $=await b.text();if($&&$.length>100)return O(i,t,p,o,_.search,l,"hit"),P(r,t,y),A($,"HIT",h.name)}}catch{}return O(i,t,p,o,_.search,l,"miss"),P(r,t,y),null}function A(e,n,t){return new Response(e,{headers:{"Content-Type":"text/html; charset=utf-8","X-Cache":n,"X-Bot":t}})}function O(e,n,t,s,r,i,u){fetch(e,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({project_id:n,url:`https://${t}${s}${r}`,domain:t,path:s,user_agent:i,phase:u})}).catch(()=>{})}function P(e,n,t){fetch(`${e}/v2/cache`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({items:[{url:t,project_id:n}]})}).catch(()=>{})}function j(e,n,t,s,r){let i={items:[{url:t,project_id:n}],ocr:s};r.length>0&&(i.path_prefixes=r),fetch(`${e}/v2/cache/ai`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(i)}).catch(()=>{})}var Y=/^\/(?:robots\.txt|llms\.txt|sitemap[^\/]*\.(?:xml|xsl))$/i,Z=/mobile|android|iphone|phone|tablet|webos|ipad/i;async function tt(e,n,t,s){let{projectId:r,siteDomain:i,prerenderDomain:u,botLogEndpoint:m,aiRender:T=!0,ocr:p=!0,aiPathPrefixes:f=[]}=s,o=new URL(e.url).pathname,l=i.replace(/^https?:\/\//,"").replace(/\/+$/,"");if(e.headers.get("x-prerender-host"))return null;if(o===`/${r}.txt`)try{let c=await fetch(`${u}${o}`,{redirect:"manual"});return new Response(c.body,{status:c.status,headers:{"Content-Type":"text/plain","Cache-Control":"public, max-age=3600"}})}catch{return new Response("",{status:404})}if(Y.test(o))try{let c=await fetch(`${u}/static${o}`,{redirect:"manual",headers:{"X-Signature":r,"X-Original-Host":l}});return new Response(c.body,{status:c.status,headers:{"Content-Type":c.headers.get("Content-Type")||"text/plain","Cache-Control":"public, max-age=3600"}})}catch{return null}let h=e.headers.get("user-agent")||"",d=R(h);if(!d||!n.V2_CACHE)return null;let y=Z.test(h)?"mobile":"desktop",x=`https://${l}${o}`,a=k.has(d.name)?"ai-user":d.isAi?"ai-crawl":"standard",b=c=>t.waitUntil(c().catch(C=>console.error("[bot-proxy]",C))),$=f.length===0||f.some(c=>o.startsWith(c));try{if(d.isAi&&T&&$){let E=await w(o,"ai"),g=await n.V2_CACHE.get(E);if(g)return b(async()=>{console.log(`[BOT] ${d.name} | ${a} | hit_ai | ${o}`),await S(m,r,l,o,h,"hit_ai"),await L(u,r,x,p,f)}),B(g,"HIT-AI",d.name);let J=await w(o,"desktop"),K=await n.V2_CACHE.get(J);return K?(b(async()=>{console.log(`[BOT] ${d.name} | ${a} | hit_desktop_fallback | ${o}`),await S(m,r,l,o,h,"hit_desktop_fallback"),await L(u,r,x,p,f)}),B(K,"HIT-DESKTOP",d.name)):(b(async()=>{console.log(`[BOT] ${d.name} | ${a} | miss_ai | ${o}`),await S(m,r,l,o,h,"miss_ai"),await Promise.all([L(u,r,x,p,f),N(u,r,x)])}),null)}let c=await w(o,y),C=await n.V2_CACHE.get(c);return C?(b(async()=>{console.log(`[BOT] ${d.name} | ${a} | hit | ${o}`),await S(m,r,l,o,h,"hit"),await N(u,r,x)}),B(C,"HIT",d.name)):(b(async()=>{console.log(`[BOT] ${d.name} | ${a} | miss | ${o}`),await S(m,r,l,o,h,"miss"),await N(u,r,x)}),null)}catch(c){return console.error("[bot-proxy]",c),null}}function B(e,n,t){return new Response(e,{headers:{"Content-Type":"text/html; charset=utf-8","X-Cache":n,"X-Bot":t}})}async function S(e,n,t,s,r,i){await fetch(e,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({project_id:n,url:`https://${t}${s}`,domain:t,path:s,user_agent:r,phase:i})})}async function N(e,n,t){await fetch(`${e}/v2/cache/kv`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({items:[{url:t,project_id:n}]})})}async function L(e,n,t,s,r){let i={items:[{url:t,project_id:n}],ocr:s};r.length>0&&(i.path_prefixes=r),await fetch(`${e}/v2/cache/kv/ai`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(i)})}export{tt as cloudflareProxy,Q as nextjsProxy};
|
package/package.json
CHANGED
package/README.md
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
# @searchos/bot-proxy
|