mcp-scraper 0.96.0 → 0.96.1

package/dist/chunk-JHT6A4U6.js

@@ -0,0 +1,11 @@
+import{c as b}from"./chunk-5LYINW7Z.js";import{g as y,i as f}from"./chunk-C7MGJUTU.js";var h={\u006b\u0065\u0072\u006e\u0065\u006c:"browserRuntime"},E=/error|message/i,x=(new RegExp("(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\\/\\/\\S*(?:dashboard|on\u006b\u0065\u0072\u006e\u0065\u006c)", "i")),T=/\b(?:ENOENT|ENOTDIR|EACCES|EPERM|EISDIR|EMFILE|ENFILE)\b|\b(?:mkdir|open|stat|lstat|readFile|writeFile)\s+['"]?(?:\/|[A-Za-z]:\\)|(?:^|[\s'"])(?:\/Users|\/home|\/tmp|\/var|\/private|\/opt|\/srv|\/root)\//i,N=new Set(["\u006b\u0065\u0072\u006e\u0065\u006c_session_id","browser_session_id","\u006b\u0065\u0072\u006e\u0065\u006cSessionId","browserSessionId","sessionId","\u006b\u0065\u0072\u006e\u0065\u006c_delete_started","session_cleanup_started","\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted","sessionCleanupStarted","\u006b\u0065\u0072\u006e\u0065\u006c_delete_succeeded","session_cleanup_succeeded","\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded","sessionCleanupSucceeded","\u006b\u0065\u0072\u006e\u0065\u006c_delete_error","session_cleanup_error","\u006b\u0065\u0072\u006e\u0065\u006cDeleteError","sessionCleanupError","\u006b\u0065\u0072\u006e\u0065\u006cProxyId","proxyId","requestedProxyIdSuffix","retrievedProxyIdSuffix","provider","providerName","service","serviceName","vendor","vendorName","organization","organizationId","accountId","browser_provider","provider_session_id","provider_session_observed_at","provider_lookup_status","provider_status","provider_error_code","provider_error_message","provider_reason_source","provider_duration_ms","provider_bandwidth_bytes","provider_captcha","provider_checked_at","provider_detail","provider_detail_json","provider_disconnect_observed","provider_disconnect_message","browserProvider","providerSessionId","providerLookupStatus","providerStatus","providerErrorCode","providerErrorMessage","providerReasonSource","providerDurationMs","providerBandwidthBytes","providerCaptcha","providerCheckedAt","providerDetail","providerDisconnectObserved","providerDisconnectMessage"]),O=new Set(["hosted_url","hostedUrl","live_view_url","liveViewUrl","cdp_ws_url","cdpWsUrl","browser_live_view_url"]),L=(new RegExp("\\b(?:wss?|https?):\\/\\/[^\\s\"'<>]*\\bon\u006b\u0065\u0072\u006e\u0065\u006c\\.com[^\\s\"'<>]*", "gi"));function k(e){return e.replace(L,"[browser-service]")}var D=new Set(["\u0062\u0072\u0069\u0067\u0068\u0074_\u0064\u0061\u0074\u0061_browser_api","\u006b\u0065\u0072\u006e\u0065\u006c","local"]),j=new Set(["pending","complete","not_found","unavailable_credentials","temporary_error","invalid_payload","not_applicable"]),q=new Set(["active","running","complete","completed","failed","closed","stopped","terminated","timed_out"]),F=new Set(["provider_session_api","client_runtime","extractor","unknown"]),U=new Set(["browser_disconnected","worker_disconnect","job_killed","cdp_cmd_timeout","network_inactivity_timeout","session_timeout","client_timeout","navigate_domains_limit","cdp_error","browser_session_interrupted","provider_detail_pending","provider_lookup_unavailable","provider_reported_no_error","provider_session_not_found","provider_invalid_json","provider_invalid_payload","provider_lookup_timeout","provider_lookup_network_error","unknown"]),M=new Set(["browser_provider","provider_session_id","provider_lookup_status","provider_status","provider_error_code","provider_error_message","provider_reason_source","provider_duration_ms","provider_bandwidth_bytes","provider_captcha","provider_checked_at","provider_detail","provider_detail_json","provider_disconnect_observed","provider_disconnect_message"]),$=512,z=320;function p(e,r){return typeof e=="string"&&r.has(e)?e:null}function H(e){return typeof e!="string"?null:U.has(e)||/^provider_http_(?:429|5\d\d)$/.test(e)?e:null}function Q(e){return typeof e!="string"||e.length<1||e.length>$||/^(?:authorization|bearer|password|secret|token|api[_-]?key|account)/i.test(e)?null:/^[A-Za-z0-9._:-]+$/.test(e)&&!e.includes("://")?e:null}function R(e){return typeof e=="number"&&Number.isSafeInteger(e)&&e>=0?e:null}function A(e){if(typeof e!="string"||e.length>40)return null;let r=Date.parse(e);return Number.isFinite(r)&&new Date(r).toISOString()===e?e:null}function C(e){if(typeof e!="string")return null;let r=e.replace(/[\u0000-\u001f\u007f]+/g," ").replace(/\s+/g," ").trim();if(!r||r.startsWith("{")||r.startsWith("["))return null;let t=r.replace(/\b(?:wss?|https?):\/\/\S+/gi,"[redacted endpoint]").replace(/\b(?:authorization|proxy-authorization)\s*:\s*(?:Bearer\s+)?\S+/gi,"[redacted credential]").replace(/\bBearer\s+\S+/gi,"[redacted credential]").replace(/\b(?:api[_-]?key|password|secret|token|account[_-]?id)\s*[=:]\s*[^\s,;]+/gi,"[redacted credential]").replace(/\b(?:account|organization)\s+(?:id\s+)?[A-Za-z0-9_-]{4,}/gi,"[redacted account]");return k(t).slice(0,z)}function w(e){return typeof e=="boolean"?e:null}function V(e){return{browser_provider:p(e.browser_provider,D),provider_session_id:Q(e.provider_session_id),provider_session_observed_at:A(e.provider_session_observed_at),provider_lookup_status:p(e.provider_lookup_status,j),provider_status:p(e.provider_status,q),provider_error_code:H(e.provider_error_code),provider_error_message:C(e.provider_error_message),provider_reason_source:p(e.provider_reason_source,F),provider_duration_ms:R(e.provider_duration_ms),provider_bandwidth_bytes:R(e.provider_bandwidth_bytes),provider_captcha:w(e.provider_captcha),provider_checked_at:A(e.provider_checked_at),provider_disconnect_observed:w(e.provider_disconnect_observed),provider_disconnect_message:C(e.provider_disconnect_message)}}function l(e,r=""){if(typeof e=="string"){let t=k(e);return E.test(r)&&(x.test(e)||T.test(e))?t=y("service_unavailable"):E.test(r)&&(t=f(t)),t}if(Array.isArray(e))return e.map(t=>l(t,r));if(e!==null&&typeof e=="object"){let t={};for(let[n,o]of Object.entries(e)){if(N.has(n))continue;let i=h[n]??n;if(O.has(n)){t[i]=null;continue}t[i]=l(o,n)}return t}return e}function Y(e){return e.map(r=>{if(r===null||typeof r!="object"||Array.isArray(r))return l(r);let t=r,n=l(t);return Object.keys(t).some(i=>M.has(i))?{...n,...V(t)}:n})}function X(e){let r=e?.diagnostics;if(!r)return e;let t=r.serpCompleteness,n=t?.queryIntegrity;return!r.debug&&!n?e:{...e,diagnostics:{...r,...r.debug?{debug:l(r.debug)}:{},...n?{serpCompleteness:{...t,queryIntegrity:{...n,outboundProviderMethod:"search"}}}:{}}}}var ee="strftime('%Y-%m-%dT%H:%M:%fZ','now')",_="julianday('now')";var oe=20,m="(json_extract(options,'$.executionOwner')='direct' AND json_extract(options,'$.serpOnly') IS NOT 1)",se=`NOT COALESCE(${m},0) AND json_extract(options,'$.syncExecutionProtocol') IS NOT 1 AND json_extract(options,'$.queueExecutionProtocol') IS NULL`,ie=`${m} AND julianday(json_extract(options,'$.paaLeaseExpiresAt')) > ${_}`,ae=`${m} AND (
+  (json_extract(options,'$.paaLeaseExpiresAt') IS NOT NULL AND COALESCE(julianday(json_extract(options,'$.paaLeaseExpiresAt')),0) <= ${_})
+  OR (json_extract(options,'$.paaLeaseExpiresAt') IS NULL AND julianday(started_at) <= julianday('now','-15 minutes'))
+)`;function ue(e){return{jobId:e.id,userId:e.user_id,leaseToken:typeof e.options.paaLeaseToken=="string"?e.options.paaLeaseToken:null,recoveryCount:Number(e.options.paaRecoveryCount??0),resultJson:e.result==null?null:JSON.stringify(e.result)}}function de(e,r){return!r||r.jobId!==e||!Number.isSafeInteger(r.userId)||r.userId<=0||!Number.isSafeInteger(r.recoveryCount)||r.recoveryCount<0||r.leaseToken!==null&&(typeof r.leaseToken!="string"||r.leaseToken.length===0)?{sql:"0",args:[]}:{sql:`user_id=? AND json_extract(options,'$.paaLeaseToken') IS ?
+    AND COALESCE(json_extract(options,'$.paaRecoveryCount'),0)=? AND json(result) IS json(?)`,args:[r.userId,r.leaseToken,r.recoveryCount,r.resultJson]}}function le(e,r,t){let n={userId:e.user_id,tool:"harvest_paa",jobId:e.id,sql:"0",args:[]},o=e.options.requestFingerprint;if(e.options.executionProtocol!==1||e.options.executionOwner!=="direct"||e.options.serpOnly===!0||typeof o!="string"||!/^[a-f0-9]{64}$/.test(o)||!r||t.userId!==e.user_id||t.key!==e.options.billingDebitKey||t.amountMc!==e.options.billingHoldMc)return n;let i=b(t);return{...n,sql:`EXISTS (SELECT 1 FROM jobs j WHERE j.id=? AND j.user_id=? AND j.query=? AND j.status='running'
+      AND json_extract(j.options,'$.executionOwner')='direct' AND json_extract(j.options,'$.serpOnly') IS NOT 1
+      AND json_extract(j.options,'$.executionProtocol')=1 AND json_extract(j.options,'$.requestFingerprint')=?
+      AND json_extract(j.options,'$.paaLeaseToken')=?
+      AND julianday(json_extract(j.options,'$.paaLeaseExpiresAt')) > ${_}
+      AND json_extract(j.options,'$.billingDebitKey')=? AND json_extract(j.options,'$.billingHoldMc')=?
+      AND json_extract(j.options,'$.billingSettledAt') IS NULL AND (${i.sql}))`,args:[e.id,e.user_id,e.query,o,r,t.key,t.amountMc,...i.args]}}import{createHash as B}from"crypto";var G=["observed_present","observed_absent","incomplete","not_requested","unknown","unsupported"],J=["provider_transport","invalid_payload","serp_readiness","query_integrity","pagination","optional_feature_parsing","unknown"],P=class extends Error{stage;failureCode;constructor(r,t,n){super(n),this.name="SerpExtractionError",this.stage=r,this.failureCode=t}};function g(e){return B("sha256").update(e).digest("hex")}function W(e){return e.normalize("NFKC").toLocaleLowerCase("en-US").split(/[^\p{L}\p{N}]+/u).filter(Boolean).slice(0,64)}function _e(e){try{let r=new URL(e);return r.hash="",r.pathname.length>1&&(r.pathname=r.pathname.replace(/\/+$/,"")),r.toString()}catch{return e.trim()}}function ge(e,r){let t=e===2?[1,2]:[1],n=r.filter(s=>s.attempted).map(s=>s.page),o=r.filter(s=>s.captured&&s.resultCount>0).map(s=>s.page),i=r.find(s=>s.failureCode);return{status:t.every(s=>o.includes(s))?"complete":"incomplete",requestedPages:t,attemptedPages:n,capturedPages:o,pageResultCounts:t.map(s=>({page:s,resultCount:r.find(S=>S.page===s)?.resultCount??0})),providerRequestCount:n.length,evidence:r,failureCode:i?.failureCode??null}}function Se(e){return e.requested?e.moduleObserved?!e.rawShapeValid||e.rawCount>0&&e.mappedCount===0?"incomplete":e.mappedCount>0?"observed_present":"observed_absent":"unknown":"not_requested"}function me(e){let r=g(e.query),t=e.finalQuery==null?null:g(e.finalQuery),n=e.locationResolutionSource==="unresolved";return{status:t!==null&&t!==r?"query_mismatch":n?"unresolved_location":"matched",queryHash:r,normalizedQueryTokens:W(e.query),canonicalLocation:e.canonicalLocation??null,locationResolutionSource:e.locationResolutionSource,locationAmbiguousCandidates:Math.max(0,Math.trunc(e.locationAmbiguousCandidates??0)),uulePresent:!!e.uule,uuleHash:e.uule?g(e.uule):null,outboundProviderMethod:e.outboundProviderMethod,finalQueryHash:t}}function ve(e){return g(JSON.stringify(Object.fromEntries(Object.entries(e).filter(([r,t])=>r!=="requestFingerprint"&&t!==void 0).sort(([r],[t])=>r.localeCompare(t)))))}function c(e){return e&&typeof e=="object"&&!Array.isArray(e)?e:{}}function v(e){return typeof e=="number"&&Number.isFinite(e)&&e>=0?Math.trunc(e):0}function u(e,r){return typeof e=="string"&&e.length<=r?e:null}function ye(e){let r=c(e),t=c(r.pagination),n=c(r.features),o=c(r.queryIntegrity),i=a=>Array.isArray(a)?a.filter(d=>d===1||d===2).slice(0,2):[],s=i(t.requestedPages);if(s.length===0)return null;let S=new Set(G),I=new Set(J);return{pagination:{status:t.status==="complete"?"complete":"incomplete",requestedPages:s,attemptedPages:i(t.attemptedPages),capturedPages:i(t.capturedPages),pageResultCounts:Array.isArray(t.pageResultCounts)?t.pageResultCounts.slice(0,2).map(a=>{let d=c(a);return{page:d.page===2?2:1,resultCount:v(d.resultCount)}}):[],providerRequestCount:v(t.providerRequestCount),failureCode:u(t.failureCode,100)},features:Object.fromEntries(["localPack","forums","videos","aiOverview","whatPeopleSaying","aiMode","paaPreview","entityIds"].map(a=>[a,S.has(String(n[a]))?n[a]:"unknown"])),queryIntegrity:{status:["matched","unresolved_location","query_mismatch"].includes(String(o.status))?o.status:"query_mismatch",queryHash:u(o.queryHash,64)??"",normalizedQueryTokens:Array.isArray(o.normalizedQueryTokens)?o.normalizedQueryTokens.slice(0,64).flatMap(a=>typeof a=="string"?[a.slice(0,100)]:[]):[],canonicalLocation:u(o.canonicalLocation,500),locationResolutionSource:u(o.locationResolutionSource,100)??"unknown",locationAmbiguousCandidates:v(o.locationAmbiguousCandidates),uulePresent:o.uulePresent===!0,uuleHash:u(o.uuleHash,64),outboundProviderMethod:"search",finalQueryHash:u(o.finalQueryHash,64)},failureStage:I.has(String(r.failureStage))?r.failureStage:null}}export{l as a,Y as b,X as c,ee as d,oe as e,m as f,se as g,ie as h,ae as i,ue as j,de as k,le as l,P as m,_e as n,ge as o,Se as p,me as q,ve as r,ye as s};