mcp-scraper 0.89.0 → 0.89.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -14
- package/README.md +5 -3
- package/dist/{analytics-repository-2BMT5JNE.js → analytics-repository-J25XR5E7.js} +1 -1
- package/dist/bin/api-server.js +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-core.js +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.js +1 -1
- package/dist/bin/paa-harvest.js +1 -1
- package/dist/chunk-3GP5CYZX.js +1 -1
- package/dist/chunk-4FROKQJN.js +1 -1
- package/dist/{chunk-TXENITMS.js → chunk-57AHYL2O.js} +1 -1
- package/dist/chunk-5H3MCTTP.js +1 -0
- package/dist/{chunk-GMWKPIYX.js → chunk-63T4W2LX.js} +182 -83
- package/dist/{chunk-WSCGYRWA.js → chunk-645LQBE7.js} +1 -1
- package/dist/{chunk-7YGVI5J4.js → chunk-6QO4F2KC.js} +3 -3
- package/dist/{chunk-M22MM4N4.js → chunk-D6IQJP64.js} +1 -1
- package/dist/{chunk-CFI6CXIV.js → chunk-DGBLGFNJ.js} +1 -1
- package/dist/{chunk-4AI7DOS7.js → chunk-FFPD2DOF.js} +1 -1
- package/dist/chunk-FXEWHK54.js +1 -0
- package/dist/{chunk-HDPYG3XV.js → chunk-IJ2AO7AO.js} +1 -1
- package/dist/{chunk-PUHFVA7P.js → chunk-MEM4D2QN.js} +4 -4
- package/dist/{chunk-E2WRWV3A.js → chunk-PRWZNHQB.js} +1 -1
- package/dist/chunk-QPWPR5XG.js +3 -3
- package/dist/{chunk-MASR22K4.js → chunk-RYNHNEF3.js} +15 -15
- package/dist/{chunk-L4CGLFPU.js → chunk-YUPLOOTB.js} +1 -1
- package/dist/{chunk-XPZVJIZ2.js → chunk-ZHORX44A.js} +1 -1
- package/dist/db-JJWJLAVQ.js +1 -0
- package/dist/{extract-bundle-SNUIHM3J.js → extract-bundle-TTSEGWYF.js} +1 -1
- package/dist/{gmail-service-BZ3H75XC.js → gmail-service-34PIJRLK.js} +1 -1
- package/dist/index.cjs +143 -44
- package/dist/index.d.cts +14 -14
- package/dist/index.d.ts +14 -14
- package/dist/index.js +1 -1
- package/dist/{lead-list-enrichment-repository-S2H3U7T7.js → lead-list-enrichment-repository-VJBBMBCG.js} +1 -1
- package/dist/{location-data-repository-OTWHWMV6.js → location-data-repository-W3LPT6G3.js} +1 -1
- package/dist/{server-RFR2A5UJ.js → server-UEY5YQL6.js} +846 -820
- package/dist/{site-extract-repository-SE776XDC.js → site-extract-repository-7W5ZUNPR.js} +1 -1
- package/dist/worker-ZOTOKG56.js +1 -0
- package/package.json +131 -17
- package/THIRD_PARTY_NOTICES.html +0 -203
- package/dist/chunk-X54CQLK2.js +0 -1
- package/dist/chunk-Z2QGQJS2.js +0 -1
- package/dist/db-F2MX63GI.js +0 -1
- package/dist/worker-XUDSM3AL.js +0 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import{a as Ge,f as de,h as le,i as De,j as Ue,l as ue}from"./chunk-
|
|
1
|
+
import{a as Ge,f as de,h as le,i as De,j as Ue,l as ue}from"./chunk-D6IQJP64.js";import{c as Pe,d as Me}from"./chunk-HUV2WTRW.js";import{W as G,e as V,q as C,t as ce,u as Re,v as ke,w as W,y as Oe}from"./chunk-MZN4U5BL.js";import{b as ae,h as se}from"./chunk-RYNHNEF3.js";import{Jb as re,M as Ne,Ob as X,s as y}from"./chunk-63T4W2LX.js";import{createHash as Te,createHmac as pt,timingSafeEqual as on}from"crypto";import{createHash as D,randomUUID as He}from"crypto";import{Client as Mt,StreamableHTTPClientTransport as Gt}from"@modelcontextprotocol/client";import{createHash as vt,randomUUID as St}from"crypto";import{z as b}from"zod";async function _n(e,t){!Number.isFinite(t)||t<=0||await ae({op:G.CONNECTED_ACCOUNT_FEE,userId:Number(e)},()=>se({vendor:"nango_connection",units:t,unitType:"connection_month"}))}async function qe(e,t){let n=[{vendor:"nango_function_run",units:t.functionRuns,unitType:"function_run"},{vendor:"nango_proxy_request",units:t.proxyRequests,unitType:"proxy_request"},{vendor:"nango_compute",units:t.computeMilliseconds/1e3,unitType:"compute_second"}];await ae({op:G.CONNECTED_USAGE,userId:Number(e)},async()=>{for(let i of n)!Number.isFinite(i.units)||i.units<=0||await se(i)})}var R="connected_usage",Ct="mastra_scheduler",xt=b.object({toolName:b.string().trim().min(1).max(200).optional(),operationKind:b.enum(["read","action","sync","export","proxy","other"]).optional(),outcome:b.enum(["success","error","partial"]).optional(),requestId:b.string().trim().min(1).max(200).optional(),occurrenceId:b.string().trim().min(1).max(200).optional(),scheduleId:b.string().trim().min(1).max(200).optional(),nangoOperationId:b.string().trim().min(1).max(200).optional(),startedAt:b.string().datetime({offset:!0}).optional(),completedAt:b.string().datetime({offset:!0}).optional()}).strict(),Tt=b.object({identity:b.string().trim().min(1).max(320),idempotencyKey:b.string().trim().min(1).max(500),ratePolicyVersion:b.string().trim().min(1).max(100),connectionId:b.string().trim().min(1).max(500),providerConfigKey:b.string().trim().min(1).max(200),functionRuns:b.number().int().safe().nonnegative(),proxyRequests:b.number().int().safe().nonnegative(),computeMilliseconds:b.number().int().safe().nonnegative(),sourceSurface:b.enum(["main_mcp","mastra_scheduler","nango_reconciliation"]).default(Ct),metadata:xt.optional()}).strict(),Et=b.object({identity:b.string().trim().min(1).max(320),ratePolicyVersion:b.string().trim().min(1).max(100)}).strict(),x=class extends Error{constructor(n,i,r,o={}){super(i);this.code=n;this.httpStatus=r;this.details=o;this.name="ConnectedUsageBillingError"}code;httpStatus;details};function ge(e){return vt("sha256").update(e).digest("hex")}function At(e){return`connected-usage:${ge(e)}`}function Nt(e){return`connected-usage-debit:${ge(e)}`}function Rt(e){return e.trim().toLowerCase()}async function Le(e){let t=await Ne(Rt(e));if(!t)throw new x("user_not_found","billing identity was not found",404);return t}function kt(e){let t={toolName:e.metadata.toolName??null,operationKind:e.metadata.operationKind??null,outcome:e.metadata.outcome??null,requestId:e.metadata.requestId??null,occurrenceId:e.metadata.occurrenceId??null,scheduleId:e.metadata.scheduleId??null,nangoOperationId:e.metadata.nangoOperationId??null,startedAt:e.metadata.startedAt??null,completedAt:e.metadata.completedAt??null};return ge(JSON.stringify({ratePolicyVersion:C,connectionId:e.connectionId,providerConfigKey:e.providerConfigKey,usage:e.usage,sourceSurface:e.sourceSurface,metadata:t}))}function K(e){let t=JSON.stringify(e);if(t.length>4e3)throw new x("invalid_request","connected usage metadata is too large",400);return t}function ye(e){try{let t=JSON.parse(e.metadata??"");if(t.contract!=="connected_usage_receipt"||t.ratePolicyVersion!==C||typeof t.fingerprint!="string"||typeof t.connectionId!="string"||typeof t.providerConfigKey!="string"||!t.usage||!t.charge)throw new Error("invalid connected usage receipt metadata");return t}catch{throw new x("idempotency_key_conflict","the idempotency key is already bound to an incompatible billing receipt",409)}}async function me(e){let t=await y().execute({sql:"SELECT id, user_id, billing_class, source_surface, status, amount_mc, metadata FROM billing_events WHERE idempotency_key = ? LIMIT 1",args:[e]});return t.rows[0]?t.rows[0]:null}async function Ke(e,t){return(await y().execute({sql:`SELECT 1
|
|
2
2
|
FROM billing_events
|
|
3
3
|
WHERE user_id = ?
|
|
4
4
|
AND billing_class = ?
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{g as _,i as c}from"./chunk-63T4W2LX.js";var g={kernel:"browserRuntime"},a=/error|message/i,w=/(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\/\/\S*(?:dashboard|onkernel)/i,E=/\b(?:ENOENT|ENOTDIR|EACCES|EPERM|EISDIR|EMFILE|ENFILE)\b|\b(?:mkdir|open|stat|lstat|readFile|writeFile)\s+['"]?(?:\/|[A-Za-z]:\\)|(?:^|[\s'"])(?:\/Users|\/home|\/tmp|\/var|\/private|\/opt|\/srv|\/root)\//i,k=new Set(["kernel_session_id","browser_session_id","kernelSessionId","browserSessionId","sessionId","kernel_delete_started","session_cleanup_started","kernelDeleteStarted","sessionCleanupStarted","kernel_delete_succeeded","session_cleanup_succeeded","kernelDeleteSucceeded","sessionCleanupSucceeded","kernel_delete_error","session_cleanup_error","kernelDeleteError","sessionCleanupError","kernelProxyId","proxyId","requestedProxyIdSuffix","retrievedProxyIdSuffix","provider","providerName","service","serviceName","vendor","vendorName","organization","organizationId","accountId","browser_provider","provider_session_id","provider_session_observed_at","provider_lookup_status","provider_status","provider_error_code","provider_error_message","provider_reason_source","provider_duration_ms","provider_bandwidth_bytes","provider_captcha","provider_checked_at","provider_detail","provider_detail_json","provider_disconnect_observed","provider_disconnect_message","browserProvider","providerSessionId","providerLookupStatus","providerStatus","providerErrorCode","providerErrorMessage","providerReasonSource","providerDurationMs","providerBandwidthBytes","providerCaptcha","providerCheckedAt","providerDetail","providerDisconnectObserved","providerDisconnectMessage"]),S=new Set(["hosted_url","hostedUrl","live_view_url","liveViewUrl","cdp_ws_url","cdpWsUrl","browser_live_view_url"]),f=/\b(?:wss?|https?):\/\/[^\s"'<>]*\bonkernel\.com[^\s"'<>]*/gi;function b(e){return e.replace(f,"[browser-service]")}var R=new Set(["bright_data_browser_api","kernel","local"]),m=new Set(["pending","complete","not_found","unavailable_credentials","temporary_error","invalid_payload","not_applicable"]),h=new Set(["active","running","complete","completed","failed","closed","stopped","terminated","timed_out"]),I=new Set(["provider_session_api","client_runtime","extractor","unknown"]),y=new Set(["browser_disconnected","worker_disconnect","job_killed","cdp_cmd_timeout","network_inactivity_timeout","session_timeout","client_timeout","navigate_domains_limit","cdp_error","browser_session_interrupted","provider_detail_pending","provider_lookup_unavailable","provider_reported_no_error","provider_session_not_found","provider_invalid_json","provider_invalid_payload","provider_lookup_timeout","provider_lookup_network_error","unknown"]),T=new Set(["browser_provider","provider_session_id","provider_lookup_status","provider_status","provider_error_code","provider_error_message","provider_reason_source","provider_duration_ms","provider_bandwidth_bytes","provider_captcha","provider_checked_at","provider_detail","provider_detail_json","provider_disconnect_observed","provider_disconnect_message"]),P=512,A=320;function t(e,r){return typeof e=="string"&&r.has(e)?e:null}function O(e){return typeof e!="string"?null:y.has(e)||/^provider_http_(?:429|5\d\d)$/.test(e)?e:null}function D(e){return typeof e!="string"||e.length<1||e.length>P||/^(?:authorization|bearer|password|secret|token|api[_-]?key|account)/i.test(e)?null:/^[A-Za-z0-9._:-]+$/.test(e)&&!e.includes("://")?e:null}function p(e){return typeof e=="number"&&Number.isSafeInteger(e)&&e>=0?e:null}function l(e){if(typeof e!="string"||e.length>40)return null;let r=Date.parse(e);return Number.isFinite(r)&&new Date(r).toISOString()===e?e:null}function u(e){if(typeof e!="string")return null;let r=e.replace(/[\u0000-\u001f\u007f]+/g," ").replace(/\s+/g," ").trim();if(!r||r.startsWith("{")||r.startsWith("["))return null;let o=r.replace(/\b(?:wss?|https?):\/\/\S+/gi,"[redacted endpoint]").replace(/\b(?:authorization|proxy-authorization)\s*:\s*(?:Bearer\s+)?\S+/gi,"[redacted credential]").replace(/\bBearer\s+\S+/gi,"[redacted credential]").replace(/\b(?:api[_-]?key|password|secret|token|account[_-]?id)\s*[=:]\s*[^\s,;]+/gi,"[redacted credential]").replace(/\b(?:account|organization)\s+(?:id\s+)?[A-Za-z0-9_-]{4,}/gi,"[redacted account]");return b(o).slice(0,A)}function v(e){return typeof e=="boolean"?e:null}function N(e){return{browser_provider:t(e.browser_provider,R),provider_session_id:D(e.provider_session_id),provider_session_observed_at:l(e.provider_session_observed_at),provider_lookup_status:t(e.provider_lookup_status,m),provider_status:t(e.provider_status,h),provider_error_code:O(e.provider_error_code),provider_error_message:u(e.provider_error_message),provider_reason_source:t(e.provider_reason_source,I),provider_duration_ms:p(e.provider_duration_ms),provider_bandwidth_bytes:p(e.provider_bandwidth_bytes),provider_captcha:v(e.provider_captcha),provider_checked_at:l(e.provider_checked_at),provider_disconnect_observed:v(e.provider_disconnect_observed),provider_disconnect_message:u(e.provider_disconnect_message)}}function i(e,r=""){if(typeof e=="string"){let o=b(e);return a.test(r)&&(w.test(e)||E.test(e))?o=_("service_unavailable"):a.test(r)&&(o=c(o)),o}if(Array.isArray(e))return e.map(o=>i(o,r));if(e!==null&&typeof e=="object"){let o={};for(let[n,d]of Object.entries(e)){if(k.has(n))continue;let s=g[n]??n;if(S.has(n)){o[s]=null;continue}o[s]=i(d,n)}return o}return e}function L(e){return e.map(r=>{if(r===null||typeof r!="object"||Array.isArray(r))return i(r);let o=r,n=i(o);return Object.keys(o).some(s=>T.has(s))?{...n,...N(o)}:n})}function C(e){let r=e?.diagnostics;return r?.debug?{...e,diagnostics:{...r,debug:i(r.debug)}}:e}export{i as a,L as b,C as c};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import{b as O}from"./chunk-KJQXUZ4Y.js";import{s as N}from"./chunk-
|
|
1
|
+
import{b as O}from"./chunk-KJQXUZ4Y.js";import{s as N}from"./chunk-63T4W2LX.js";import{createHash as M,randomUUID as z}from"crypto";var D="us_zip_groups",K="us_census_places",P=25e3,k=["AL","AK","AZ","AR","CA","CO","CT","DE","DC","FL","GA","HI","ID","IL","IN","IA","KS","KY","LA","ME","MD","MA","MI","MN","MS","MO","MT","NE","NV","NH","NJ","NM","NY","NC","ND","OH","OK","OR","PA","RI","SC","SD","TN","TX","UT","VT","VA","WA","WV","WI","WY"],$={AL:"01",AK:"02",AZ:"04",AR:"05",CA:"06",CO:"08",CT:"09",DE:"10",DC:"11",FL:"12",GA:"13",HI:"15",ID:"16",IL:"17",IN:"18",IA:"19",KS:"20",KY:"21",LA:"22",ME:"23",MD:"24",MA:"25",MI:"26",MN:"27",MS:"28",MO:"29",MT:"30",NE:"31",NV:"32",NH:"33",NJ:"34",NM:"35",NY:"36",NC:"37",ND:"38",OH:"39",OK:"40",OR:"41",PA:"42",RI:"44",SC:"45",SD:"46",TN:"47",TX:"48",UT:"49",VT:"50",VA:"51",WA:"53",WV:"54",WI:"55",WY:"56"},x=new Set(k),A=200,H=2e3,v=null,w=null;function b(t){return t.toLowerCase().replace(/[^a-z0-9]+/g," ").trim()}function Y(t){if(t==null)return[];try{let o=JSON.parse(String(t));return Array.isArray(o)?o.map(String):[]}catch{return[]}}function C(t){return{id:String(t.id),kind:String(t.dataset_kind),sourceUrl:t.source_url==null?null:String(t.source_url),sha256:String(t.sha256),rowCount:Number(t.row_count??0),groupCount:Number(t.group_count??0),stateCount:Number(t.state_count??0),sourceRowCount:Number(t.source_row_count??t.row_count??0),rejectedRowCount:Number(t.rejected_row_count??0),zipCount:Number(t.zip_count??0),coveredStates:Y(t.covered_states_json),coverageComplete:Number(t.coverage_complete??0)===1,status:String(t.status),createdAt:String(t.created_at),completedAt:t.completed_at==null?null:String(t.completed_at),error:t.error==null?null:String(t.error)}}function F(t){return`${K}:${t}`}function X(t){let o=t?.trim();if(!o)return null;if(o.length>H)throw new Error(`sourceUrl must be ${H} characters or fewer`);let a;try{a=new URL(o)}catch{throw new Error("sourceUrl must be a valid URL")}return a.username="",a.password="",a.search="",a.hash="",a.toString()}function q(t){let o=t.trim();return/^\d{1,5}$/.test(o)?o.padStart(5,"0"):null}function f(t){let o=t.trim().toUpperCase();return x.has(o)?o:null}function y(t,o){for(let a of o){let r=t[a];if(r!==void 0)return r}return""}function g(t){if(t===void 0||t.trim()==="")return null;let o=Number(t);return Number.isFinite(o)&&o>=0?Math.trunc(o):null}function V(t){return/^Nashville-Davidson metropolitan government/i.test(t)?"Nashville":t.replace(/\s+(city|town|village|municipality|borough)$/i,"").trim()}function B(t,o){let a=O(t.replace(/^\uFEFF/,""));if(a.length===0)throw new Error("Census place CSV contains no data rows.");let r=new Set(Object.keys(a[0]??{}).map(e=>e.trim())),u=["SUMLEV","STATE","NAME","POPESTIMATE2020","POPESTIMATE2021","POPESTIMATE2022","POPESTIMATE2023","POPESTIMATE2024","POPESTIMATE2025"].filter(e=>!r.has(e));if(u.length>0)throw new Error(`Census place CSV is missing required columns: ${u.join(", ")}.`);let d=a.filter(e=>e.SUMLEV?.trim()==="162");if(o){let e=o.padStart(2,"0"),s=d.filter(l=>l.STATE?.trim().padStart(2,"0")!==e);if(s.length>0){let l=[...new Set(s.map(c=>c.STATE?.trim()||"(blank)"))].slice(0,5);throw new Error(`Census place CSV STATE does not match requested FIPS ${e}; received ${l.join(", ")}.`)}}let E=d.map(e=>{let s=e.NAME?.trim()??"";if(!s)return null;let l=V(s),c=b(l);if(!c)return null;let T={2020:g(e.POPESTIMATE2020),2021:g(e.POPESTIMATE2021),2022:g(e.POPESTIMATE2022),2023:g(e.POPESTIMATE2023),2024:g(e.POPESTIMATE2024),2025:g(e.POPESTIMATE2025)};return Object.values(T).every(m=>m===null)?null:{city:l,cityKey:c,censusName:s,estimatesBase2020:g(e.ESTIMATESBASE2020),populations:T}}).filter(e=>e!==null),p=new Map,i=e=>e.populations[2025]??e.populations[2024]??e.populations[2023]??e.populations[2022]??e.populations[2021]??e.populations[2020]??e.estimatesBase2020??-1;for(let e of E){let s=p.get(e.cityKey);(!s||i(e)>i(s)||i(e)===i(s)&&e.censusName.localeCompare(s.censusName)<0)&&p.set(e.cityKey,e)}let _=[...p.values()].sort((e,s)=>e.city.localeCompare(s.city));if(_.length===0)throw new Error("Census place CSV did not contain any valid place population rows.");return{sourceRowCount:a.length,places:_}}function W(t){let o=O(t.replace(/^\uFEFF/,""));if(o.length===0)throw new Error("Location ZIP CSV contains no data rows.");let a=o[0]??{},r=new Set(Object.keys(a).map(s=>s.trim().toLowerCase())),n=["state_abbr","state","state_id"].some(s=>r.has(s)),u=["city","primary_city"].some(s=>r.has(s)),d=["zipcode","zip","zip_code"].some(s=>r.has(s));if(!n||!u||!d)throw new Error("Location ZIP CSV must include state_abbr/state, city, and zipcode/zip columns.");let E=new Map,p=0,i=0,_=new Set,e=new Set;for(let s of o){let l=Object.fromEntries(Object.entries(s).map(([G,j])=>[G.trim().toLowerCase(),j])),c=f(y(l,["state_abbr","state","state_id"])),T=y(l,["city","primary_city"]).trim(),m=q(y(l,["zipcode","zip","zip_code"])),S=y(l,["county","county_name"]).trim();if(!c||!x.has(c)||!T||!m){i+=1;continue}let I=b(T);if(!I){i+=1;continue}let U=`${c}\0${I}`,L=E.get(U);L||(L={city:T,cityKey:I,state:c,zips:new Set,counties:new Set},E.set(U,L)),L.zips.add(m),S&&L.counties.add(S),_.add(c),e.add(m),p+=1}if(E.size===0)throw new Error("Location ZIP CSV did not contain any valid US city/ZIP rows.");return{sourceRowCount:o.length,rowCount:p,rejectedRowCount:i,zipCount:e.size,stateCount:_.size,coveredStates:[..._].sort(),groups:[...E.values()].map(s=>({city:s.city,cityKey:s.cityKey,state:s.state,zips:[...s.zips].sort(),counties:[...s.counties].sort()})).sort((s,l)=>s.state.localeCompare(l.state)||s.city.localeCompare(l.city))}}function J(t){let o=new Set(t.coveredStates),a=k.filter(n=>!o.has(n)),r=[];if(a.length>0&&r.push(`missing required states: ${a.join(", ")}`),t.zipCount<P&&r.push(`only ${t.zipCount} unique ZIPs; at least ${P} are required`),r.length>0)throw new Error(`Location ZIP snapshot is not nationwide and was not activated (${r.join("; ")}; ${t.rejectedRowCount} rejected rows).`)}async function R(){let t=N();v!==t&&(v=t,w=null),w||(w=(async()=>{await t.execute(`
|
|
2
2
|
CREATE TABLE IF NOT EXISTS location_data_imports (
|
|
3
3
|
id TEXT PRIMARY KEY,
|
|
4
4
|
dataset_kind TEXT NOT NULL,
|