mcp-scraper 0.90.3 → 0.90.5

package/dist/{site-extract-repository-T23S2T7L.js → site-extract-repository-IZT3NTWY.js}

@@ -1 +1 @@
-import{A as s,B as t,C as u,D as v,E as w,F as x,G as y,H as z,I as A,J as B,K as C,L as D,M as E,i as a,j as b,k as c,l as d,m as e,n as f,o as g,p as h,q as i,r as j,s as k,t as l,u as m,v as n,w as o,x as p,y as q,z as r}from"./chunk-ZMLCK45S.js";import"./chunk-TMB56NCA.js";import"./chunk-YEFXVTMJ.js";import"./chunk-7BEWBRMT.js";export{a as XRAY_ENTITLED_EXTRACT_BILLING_CLASS,q as abandonExtractSettlement,p as claimFailedExtractJobForRefinalize,z as completeExtractJob,w as countSuccessfulPages,d as createExtractJob,f as createOrGetExtractJob,b as extractJobLimitInfo,A as failExtractJob,o as failStaleRunningExtractJob,B as failUnfundedExtractJob,E as finalizeXRayEntitledExtractJobCharge,y as finishExtractJob,g as getExtractJob,e as getExtractJobByIdempotencyKey,v as getExtractedImageLinks,u as getExtractedPages,x as getExtractedUrls,h as listExtractJobs,t as listExtractPages,j as listFundedPendingExtractJobs,n as listStaleRunningExtractJobs,i as listUnsettledExtractJobs,k as markExtractJobDispatchAttempt,l as recordExtractJobDispatchFailure,m as recordExtractSettlementFailure,s as saveExtractPages,C as setExtractJobPublicError,r as setExtractJobTotal,D as settleExtractJob,c as terminalExtractJobStatus};
+import{A as s,B as t,C as u,D as v,E as w,F as x,G as y,H as z,I as A,J as B,K as C,L as D,M as E,i as a,j as b,k as c,l as d,m as e,n as f,o as g,p as h,q as i,r as j,s as k,t as l,u as m,v as n,w as o,x as p,y as q,z as r}from"./chunk-XSYLDTMI.js";import"./chunk-TMB56NCA.js";import"./chunk-YEFXVTMJ.js";import"./chunk-KKHTMYCD.js";export{a as XRAY_ENTITLED_EXTRACT_BILLING_CLASS,q as abandonExtractSettlement,p as claimFailedExtractJobForRefinalize,z as completeExtractJob,w as countSuccessfulPages,d as createExtractJob,f as createOrGetExtractJob,b as extractJobLimitInfo,A as failExtractJob,o as failStaleRunningExtractJob,B as failUnfundedExtractJob,E as finalizeXRayEntitledExtractJobCharge,y as finishExtractJob,g as getExtractJob,e as getExtractJobByIdempotencyKey,v as getExtractedImageLinks,u as getExtractedPages,x as getExtractedUrls,h as listExtractJobs,t as listExtractPages,j as listFundedPendingExtractJobs,n as listStaleRunningExtractJobs,i as listUnsettledExtractJobs,k as markExtractJobDispatchAttempt,l as recordExtractJobDispatchFailure,m as recordExtractSettlementFailure,s as saveExtractPages,C as setExtractJobPublicError,r as setExtractJobTotal,D as settleExtractJob,c as terminalExtractJobStatus};

package/dist/{stripe-event-worker-HVCB6SF3.js → stripe-event-worker-5QNWGHVK.js}

@@ -1 +1 @@
-import{fb as S,hb as h}from"./chunk-HJ7OGU2J.js";import"./chunk-WTOKMWDS.js";import"./chunk-HR2A2WTL.js";import"./chunk-2GRE4BAO.js";import"./chunk-D6BTAT5Y.js";import"./chunk-YEFXVTMJ.js";import"./chunk-DKQZNSYM.js";import{H as d,I as u,J as l,K as f,L as v}from"./chunk-7BEWBRMT.js";var g=12;function b(e){return e&&typeof e=="object"&&"code"in e&&typeof e.code=="string"?e.code:e instanceof Error?e.name:"unknown_error"}function k(e){return Math.min(21600,30*2**Math.max(0,e-1))}async function x(e={}){let w=e.stripeClient??S,E=Math.max(1,Math.min(100,e.limit??100)),r=500,i=0,s=0,c=0,m=0;for(;i<r;){let a=await d(Math.min(E,r-i),e.leaseSeconds??300);if(a.length===0)break;i+=a.length;for(let t of a)try{let n=await w.events.retrieve(t.eventId);await h(n),await u(t.eventId,t.token)&&(s+=1)}catch(n){let p=n instanceof Error?n.message:String(n),o=b(n);t.attemptCount>=g||o==="resource_missing"?await f(t.eventId,t.token,p,o)&&(m+=1):await l(t.eventId,t.token,p,o,k(t.attemptCount))&&(c+=1)}}return{claimed:i,processed:s,retried:c,deadLettered:m,capped:i>=r,health:await v()}}export{x as drainStripeEventQueue};
+import{fb as S,hb as h}from"./chunk-5RTZPHIY.js";import"./chunk-NC32JCZT.js";import"./chunk-4OTWTGKC.js";import"./chunk-Q5SUDDSQ.js";import"./chunk-D6BTAT5Y.js";import"./chunk-YEFXVTMJ.js";import"./chunk-56UT5M5Y.js";import{H as d,I as u,J as l,K as f,L as v}from"./chunk-KKHTMYCD.js";var g=12;function b(e){return e&&typeof e=="object"&&"code"in e&&typeof e.code=="string"?e.code:e instanceof Error?e.name:"unknown_error"}function k(e){return Math.min(21600,30*2**Math.max(0,e-1))}async function x(e={}){let w=e.stripeClient??S,E=Math.max(1,Math.min(100,e.limit??100)),r=500,i=0,s=0,c=0,m=0;for(;i<r;){let a=await d(Math.min(E,r-i),e.leaseSeconds??300);if(a.length===0)break;i+=a.length;for(let t of a)try{let n=await w.events.retrieve(t.eventId);await h(n),await u(t.eventId,t.token)&&(s+=1)}catch(n){let p=n instanceof Error?n.message:String(n),o=b(n);t.attemptCount>=g||o==="resource_missing"?await f(t.eventId,t.token,p,o)&&(m+=1):await l(t.eventId,t.token,p,o,k(t.attemptCount))&&(c+=1)}}return{claimed:i,processed:s,retried:c,deadLettered:m,capped:i>=r,health:await v()}}export{x as drainStripeEventQueue};

package/dist/worker-Z27YG7TE.js

@@ -0,0 +1 @@
+import{j as R,k as g,l as P,m as _,o as w,q as v,s as C,t as h}from"./chunk-X4YCWF4B.js";import"./chunk-XG6GCEUE.js";import{w as A}from"./chunk-ZJOD3YKH.js";import"./chunk-M5QHXNFZ.js";import{b as f,c as D}from"./chunk-AAJG5OHF.js";import{a as b,b as y}from"./chunk-YEFXVTMJ.js";import{a as O}from"./chunk-4FROKQJN.js";import{b as k}from"./chunk-56UT5M5Y.js";import"./chunk-2TZBO52D.js";import"./chunk-2PXP7TMY.js";import{Mb as o,Qb as d,Sb as m,Ua as c,gb as H,ib as p,na as u}from"./chunk-KKHTMYCD.js";var S=2,M=0;function N(e){if(!e||typeof e!="object")return 0;let t=e;return typeof t.totalQuestions=="number"?t.totalQuestions:Array.isArray(t.flat)?t.flat.length:0}function x(e){return b.paa_base+Math.max(1,e)*b.paa}async function K(e){M++;try{let t=typeof e.options=="string"?JSON.parse(e.options):e.options,r={value:null},i=w(e.id),n=await k({op:t.serpOnly?"serp":"paa",userId:Number(e.user_id),headlessSentOut:r,operationRunId:i,operationAccountScope:"production-default"},()=>A({...t,\u006b\u0065\u0072\u006e\u0065\u006cApiKey:O(),headless:!0,format:"json",outputDir:"/tmp/paa-output-api",onAttemptEvent:C(e.id,e.user_id,i)}));await H(e.id,n),await v(i,"succeeded");let l=await u(e.id,e.user_id);if(typeof t.billingHoldMc=="number"&&t.billingDebitKey){let s=t.serpOnly?y(r.value):x(N(n));await m(Number(e.user_id),t.billingDebitKey,s,t.serpOnly?"serp_refund":"paa_refund",t.serpOnly?"SERP search settlement":"PAA harvest settlement",t.serpOnly?"serp_search":"paa_harvest")}else if(!t.serpOnly&&typeof t.billingHoldMc=="number"){let s=x(N(n)),a=t.billingHoldMc-s;a>0?await o(e.user_id,a,"paa_refund","overestimate refund"):a<0&&await d(e.user_id,-a,"paa",t.query??e.query)}else if(t.serpOnly&&typeof t.billingHoldMc=="number"){let s=y(r.value),a=t.billingHoldMc-s;a>0?await o(e.user_id,a,"serp_refund","headless-mode pricing settle"):a<0&&await d(e.user_id,-a,"serp",t.query??e.query)}e.callback_url&&await h(e.callback_url,{job_id:e.id,status:"done",result:D(n),attempts:f(l)})}catch(t){console.error("[harvest/worker] failed",{jobId:e.id,error:t instanceof Error?t.message:String(t)});let r=R(t);await v(w(e.id),r.error_code==="harvest_timeout"?"timed_out":r.terminalStatus,r.error_code);let i=typeof e.options=="string"?JSON.parse(e.options):e.options,n=typeof i.billingHoldMc=="number"&&i.billingHoldMc>0;await p(e.id,g(r),_(r,{chargeStatus:n?"refund_pending":"not_charged"}));let l=await u(e.id,e.user_id);try{n&&(i.billingDebitKey?await m(Number(e.user_id),i.billingDebitKey,0,i.serpOnly?"serp_refund":"paa_refund","failed call",i.serpOnly?"serp_search":"paa_harvest"):await o(e.user_id,i.billingHoldMc,"refund","failed call"),await p(e.id,g(r),_(r,{chargeStatus:"refunded"})))}catch{}e.callback_url&&await h(e.callback_url,{job_id:e.id,status:"failed",...P(r),attempts:f(l)})}finally{M--}}async function J(){let e=await c();if(!e)return{claimed:!1};let t=Date.now();return await K(e),{claimed:!0,jobId:e.id,completed:!0,durationMs:Date.now()-t}}async function X(e){let t=[];for(let r=0;r<e.maxJobs&&!(Date.now()>=e.deadlineMs);r++){let i=await J();if(t.push(i),!i.claimed)break}return t}function $(){setInterval(async()=>{if(M>=S)return;let e=await c();e&&K(e)},2e3),console.log(`[worker] started \u2014 polling every 2s, max ${S} concurrent`)}export{X as drainQueue,$ as startWorker,J as tickOnce};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcp-scraper",
-  "version": "0.90.3",
+  "version": "0.90.5",
   "description": "MCP server for MCP Scraper web intelligence tools",
   "type": "module",
   "main": "./dist/index.cjs",

package/dist/chunk-PHBU6X5B.js

@@ -1 +0,0 @@
-import{a as m}from"./chunk-XG6GCEUE.js";import{b as n,g as o,h as i,ha as p,ia as d,j as l,ka as b,m as u,o as c}from"./chunk-7BEWBRMT.js";function f(e){return e instanceof Error?e.message:String(e)}function y(e,t){return e instanceof DOMException&&(e.name==="TimeoutError"||e.name==="AbortError")?!0:/timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(t)}function g(e){return/captcha|recaptcha|unusual traffic|google\.com\/sorry|blocked/i.test(e)}function S(e){return/ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(e)}function E(e){return/proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(e)}function h(e){return/(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\/\/\S*dashboard/i.test(e)}function v(e){return/browser (?:has been )?closed|context (?:has been )?closed|target page, context or browser has been closed|session closed|page has been closed/i.test(e)}function x(e){let t=f(e);return e instanceof u?{error_code:"request_aborted",error_type:"request_aborted",message:o("request_aborted"),retryable:!0,httpStatus:408,terminalStatus:"cancelled"}:e instanceof l||g(t)?{error_code:"captcha_exhausted",error_type:"captcha",message:o("captcha_exhausted"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:m(e,t)?{error_code:"vendor_unavailable",error_type:"service_unavailable",message:o("vendor_unavailable"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:e instanceof c?{error_code:"location_mismatch",error_type:"location_mismatch",message:o("location_mismatch"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:S(t)?{error_code:"proxy_tunnel_failed",error_type:"connection",message:o("proxy_tunnel_failed"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:E(t)?{error_code:"proxy_unavailable",error_type:"connection",message:o("proxy_unavailable"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:v(t)?{error_code:"browser_session_interrupted",error_type:"extraction",message:o("browser_session_interrupted"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:y(e,t)?{error_code:"harvest_timeout",error_type:"timeout",message:o("harvest_timeout"),retryable:!0,httpStatus:504,terminalStatus:"failed"}:h(t)?{error_code:"service_unavailable",error_type:"service_unavailable",message:n,retryable:!0,httpStatus:503,terminalStatus:"failed"}:{error_code:"extraction_failed",error_type:"extraction",message:n,retryable:!1,httpStatus:500,terminalStatus:"failed"}}function w(e){return JSON.stringify({error_code:e.error_code,error_type:e.error_type,message:e.message,retryable:e.retryable})}function _(e,t={}){let r=i({errorCode:e.error_code,errorType:e.error_type,retryable:e.retryable,retryAfterSeconds:t.retryAfterSeconds,chargeStatus:t.chargeStatus,details:t.details});return{error:r.message,...r}}function A(e,t={}){let{error:r,...a}=_(e,t);return a}function C(e,t={}){let r=x(e);return{body:_(r,t),status:r.httpStatus}}function D(e,t){return async r=>{if(r.type==="started"){await p({jobId:e,userId:t,attemptNumber:r.attemptNumber,maxAttempts:r.maxAttempts,query:r.query,location:r.location,maxQuestions:r.maxQuestions,startedAt:r.startedAt});return}if(r.type==="provider_session_started"){await d({jobId:e,attemptNumber:r.attemptNumber,browserProvider:r.provider,providerSessionId:r.providerSessionId,observedAt:r.observedAt});return}await b({jobId:e,attemptNumber:r.attemptNumber,outcome:r.outcome,\u006b\u0065\u0072\u006e\u0065\u006cSessionId:r.\u006b\u0065\u0072\u006e\u0065\u006cSessionId,questionCount:r.questionCount,durationMs:r.durationMs,error:r.error,willRetry:r.willRetry,\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted:r.cleanup.\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted,\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded:r.cleanup.\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded,\u006b\u0065\u0072\u006e\u0065\u006cDeleteError:r.cleanup.\u006b\u0065\u0072\u006e\u0065\u006cDeleteError,browserCloseSucceeded:r.cleanup.browserCloseSucceeded,browserCloseError:r.cleanup.browserCloseError,browserProvider:r.cleanup.provider??null,providerSessionId:r.cleanup.providerSessionId??null,providerLookupStatus:r.cleanup.providerSessionId?"pending":"not_applicable",providerErrorCode:r.cleanup.providerDisconnectObserved?"browser_session_interrupted":null,providerErrorMessage:r.cleanup.providerDisconnectMessage??null,providerReasonSource:r.cleanup.providerDisconnectObserved?"client_runtime":null,providerDisconnectObserved:r.cleanup.providerDisconnectObserved??!1,providerDisconnectMessage:r.cleanup.providerDisconnectMessage??null,debug:r.debug,completedAt:r.completedAt})}}async function I(e,t,r=3){for(let a=1;a<=r;a++){try{let s=await fetch(e,{method:"POST",headers:{"content-type":"application/json"},body:JSON.stringify(t),signal:AbortSignal.timeout(1e4)});if(s.ok)return;console.warn(`[webhook] attempt ${a} \u2192 ${s.status} from ${e}`)}catch(s){console.warn(`[webhook] attempt ${a} failed:`,s instanceof Error?s.message:s)}a<r&&await new Promise(s=>setTimeout(s,1e3*a*2))}console.error(`[webhook] gave up after ${r} attempts for ${e}`)}export{x as a,w as b,_ as c,A as d,C as e,D as f,I as g};

package/dist/chunk-TLF42CG7.js

@@ -1 +0,0 @@
-var o="0.90.3";export{o as a};

package/dist/worker-KFILERH3.js

@@ -1 +0,0 @@
-import{a as O,b as y,c as D,d as g,f as A,g as _}from"./chunk-PHBU6X5B.js";import"./chunk-XG6GCEUE.js";import{w as k}from"./chunk-GW6ITK76.js";import"./chunk-M5QHXNFZ.js";import{b as f,c as H}from"./chunk-7AC66V3T.js";import{a as m,b}from"./chunk-YEFXVTMJ.js";import{a as v}from"./chunk-4FROKQJN.js";import{b as M}from"./chunk-DKQZNSYM.js";import"./chunk-2TZBO52D.js";import"./chunk-2PXP7TMY.js";import{Mb as l,Qb as p,Sb as d,Ua as u,gb as h,ib as c,na as o}from"./chunk-7BEWBRMT.js";var P=2,w=0;function C(e){if(!e||typeof e!="object")return 0;let t=e;return typeof t.totalQuestions=="number"?t.totalQuestions:Array.isArray(t.flat)?t.flat.length:0}function R(e){return m.paa_base+Math.max(1,e)*m.paa}async function N(e){w++;try{let t=typeof e.options=="string"?JSON.parse(e.options):e.options,r={value:null},i=await M({op:t.serpOnly?"serp":"paa",userId:Number(e.user_id),headlessSentOut:r},()=>k({...t,\u006b\u0065\u0072\u006e\u0065\u006cApiKey:v(),headless:!0,format:"json",outputDir:"/tmp/paa-output-api",onAttemptEvent:A(e.id,e.user_id)}));await h(e.id,i);let s=await o(e.id,e.user_id);if(typeof t.billingHoldMc=="number"&&t.billingDebitKey){let n=t.serpOnly?b(r.value):R(C(i));await d(Number(e.user_id),t.billingDebitKey,n,t.serpOnly?"serp_refund":"paa_refund",t.serpOnly?"SERP search settlement":"PAA harvest settlement",t.serpOnly?"serp_search":"paa_harvest")}else if(!t.serpOnly&&typeof t.billingHoldMc=="number"){let n=R(C(i)),a=t.billingHoldMc-n;a>0?await l(e.user_id,a,"paa_refund","overestimate refund"):a<0&&await p(e.user_id,-a,"paa",t.query??e.query)}else if(t.serpOnly&&typeof t.billingHoldMc=="number"){let n=b(r.value),a=t.billingHoldMc-n;a>0?await l(e.user_id,a,"serp_refund","headless-mode pricing settle"):a<0&&await p(e.user_id,-a,"serp",t.query??e.query)}e.callback_url&&await _(e.callback_url,{job_id:e.id,status:"done",result:H(i),attempts:f(s)})}catch(t){console.error("[harvest/worker] failed",{jobId:e.id,error:t instanceof Error?t.message:String(t)});let r=O(t),i=typeof e.options=="string"?JSON.parse(e.options):e.options,s=typeof i.billingHoldMc=="number"&&i.billingHoldMc>0;await c(e.id,y(r),g(r,{chargeStatus:s?"refund_pending":"not_charged"}));let n=await o(e.id,e.user_id);try{s&&(i.billingDebitKey?await d(Number(e.user_id),i.billingDebitKey,0,i.serpOnly?"serp_refund":"paa_refund","failed call",i.serpOnly?"serp_search":"paa_harvest"):await l(e.user_id,i.billingHoldMc,"refund","failed call"),await c(e.id,y(r),g(r,{chargeStatus:"refunded"})))}catch{}e.callback_url&&await _(e.callback_url,{job_id:e.id,status:"failed",...D(r),attempts:f(n)})}finally{w--}}async function x(){let e=await u();if(!e)return{claimed:!1};let t=Date.now();return await N(e),{claimed:!0,jobId:e.id,completed:!0,durationMs:Date.now()-t}}async function W(e){let t=[];for(let r=0;r<e.maxJobs&&!(Date.now()>=e.deadlineMs);r++){let i=await x();if(t.push(i),!i.claimed)break}return t}function B(){setInterval(async()=>{if(w>=P)return;let e=await u();e&&N(e)},2e3),console.log(`[worker] started \u2014 polling every 2s, max ${P} concurrent`)}export{W as drainQueue,B as startWorker,x as tickOnce};