mcp-scraper 0.89.8 → 0.90.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +49 -17
- package/README.md +3 -3
- package/THIRD_PARTY_NOTICES.html +203 -0
- package/dist/bin/api-server.js +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-core.js +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.js +1 -1
- package/dist/bin/paa-harvest.js +1 -1
- package/dist/chunk-2PXP7TMY.js +1 -0
- package/dist/chunk-2TZBO52D.js +1 -1
- package/dist/{chunk-Q4A2KIPZ.js → chunk-3R5QLQT3.js} +1 -1
- package/dist/chunk-45Z77CM7.js +1 -0
- package/dist/chunk-4FROKQJN.js +1 -1
- package/dist/chunk-7AC66V3T.js +1 -1
- package/dist/chunk-7BEWBRMT.js +15 -15
- package/dist/chunk-DKQZNSYM.js +15 -15
- package/dist/{chunk-52XUWR3X.js → chunk-HJ7OGU2J.js} +80 -80
- package/dist/chunk-M5QHXNFZ.js +3 -3
- package/dist/{chunk-KX72IORC.js → chunk-M7EXIKT7.js} +8 -8
- package/dist/chunk-PHBU6X5B.js +1 -1
- package/dist/{chunk-AGK756UE.js → chunk-RAG2WBBL.js} +184 -184
- package/dist/{chunk-TI3KH3YW.js → chunk-SWKUC7YD.js} +1 -1
- package/dist/{chunk-5B42QVDC.js → chunk-WTOKMWDS.js} +1 -1
- package/dist/{chunk-2BS7MMOS.js → chunk-XGZJGKEG.js} +1 -1
- package/dist/chunk-YEFXVTMJ.js +1 -0
- package/dist/{chunk-VT6SLBOU.js → chunk-ZMLCK45S.js} +1 -1
- package/dist/{extract-bundle-DI7OVLT5.js → extract-bundle-7BI2K26D.js} +1 -1
- package/dist/{gmail-service-ASX7YBC3.js → gmail-service-Y2UZX3YW.js} +1 -1
- package/dist/index.cjs +58 -58
- package/dist/index.d.cts +30 -14
- package/dist/index.d.ts +30 -14
- package/dist/index.js +1 -1
- package/dist/{server-PQB2EITA.js → server-TI7YYGK5.js} +563 -563
- package/dist/{site-extract-repository-WADCAIYB.js → site-extract-repository-T23S2T7L.js} +1 -1
- package/dist/{stripe-event-worker-ZPNWXX2T.js → stripe-event-worker-HVCB6SF3.js} +1 -1
- package/dist/worker-IUJVVXIG.js +1 -0
- package/package.json +17 -137
- package/dist/chunk-E4ME7J6G.js +0 -1
- package/dist/chunk-MM6QBDPC.js +0 -1
- package/dist/chunk-Z3WXA4ZU.js +0 -1
- package/dist/worker-GTIKQJIN.js +0 -1
package/dist/chunk-7BEWBRMT.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import{createClient as Be}from"@libsql/client/http";import{createHash as U,randomBytes as P,randomUUID as g,scryptSync as pe,timingSafeEqual as Ge}from"crypto";import{z as p}from"zod";var D="2026-06-26 21:58:03";var dt="Google returned a CAPTCHA. Run with --headless=false to re-warm the browser profile, then retry.",xe="This operation is temporarily unavailable. Please retry later.",ye=["request_aborted","captcha_exhausted","captcha_or_blocked","location_mismatch","proxy_tunnel_failed","proxy_unavailable","harvest_timeout","mcp_request_timeout","mcp_http_error","extraction_failed","concurrency_limit_exceeded","unauthorized","forbidden","invalid_request","rate_limited","upstream_rate_limited","insufficient_balance","service_unavailable","response_lost","vendor_unavailable","page_not_found","page_forbidden","page_rate_limited","page_server_error","page_http_error","bot_check_unresolved","page_too_large","page_unreachable","browser_session_interrupted","site_export_not_found","site_export_format_unavailable","site_export_read_failed","site_export_image_not_found","image_url_invalid","image_url_not_https","image_host_unresolvable","image_host_blocked","image_redirect_invalid","image_source_unavailable","image_too_large","image_redirect_limit","image_base64_invalid","image_source_missing","image_source_ambiguous","image_source_type_invalid","image_artifact_not_found","image_page_too_large","page_image_not_found","page_image_candidates_unusable","image_format_unsupported","image_storage_unconfigured","featured_image_unhostable"],Ie=["request_aborted","captcha","location_mismatch","connection","timeout","extraction","concurrency_limit","unauthorized","forbidden","invalid_request","rate_limit","billing","service_unavailable"];function w(e){return e?JSON.stringify(e):null}function ne(e){if(e==null)return null;try{let t=typeof e=="string"?JSON.parse(e):e;if(!t||typeof t!="object"||Array.isArray(t))return null;let r=t;if(typeof r.error_code!="string"||!ie.has(r.error_code))return null;let s=Pe({errorCode:r.error_code,errorType:r.error_type,retryable:r.retryable,retryAfterSeconds:r.retry_after_seconds,chargeStatus:r.charge_status,details:r.details});if(typeof r.message=="string"){let i=ae(r.message).trim().slice(0,1e3);i&&!/https?:\/\//i.test(i)&&(s.message=i)}return s}catch{return null}}var ie=new Set(ye),Ce=new Set(Ie),Ue=new Set(["active","limit","operation","upgrade_url","topup_url","upgrade_command","slots_per_pack","pack_price_usd","balance_credits","required_credits","retry_guidance","http_status"]),De=12,ke=500;function W(e){return typeof e=="string"&&ie.has(e)?e:"service_unavailable"}function ve(e){if(!e||typeof e!="object"||Array.isArray(e))return;let t={};for(let[r,s]of Object.entries(e)){if(Object.keys(t).length>=De)break;if(Ue.has(r))if(typeof s=="string"){if((r==="upgrade_url"||r==="topup_url")&&!Xe(s))continue;let i=ae(s).slice(0,ke);i&&(t[r]=i)}else(typeof s=="number"&&Number.isFinite(s)||typeof s=="boolean"||s===null)&&(t[r]=s)}return Object.keys(t).length>0?t:void 0}function Xe(e){if(e.startsWith("/")&&!e.startsWith("//"))return!0;try{let t=new URL(e);return t.protocol==="https:"&&(t.hostname==="mcpscraper.dev"||t.hostname.endsWith(".mcpscraper.dev"))}catch{return!1}}function Me(e){let t=W(e);switch(t){case"captcha_exhausted":case"captcha_or_blocked":return"captcha";case"proxy_tunnel_failed":case"proxy_unavailable":case"response_lost":return"connection";case"harvest_timeout":case"mcp_request_timeout":return"timeout";case"concurrency_limit_exceeded":return"concurrency_limit";case"rate_limited":case"upstream_rate_limited":return"rate_limit";case"extraction_failed":case"page_not_found":case"page_forbidden":case"page_rate_limited":case"page_server_error":case"page_http_error":case"bot_check_unresolved":case"page_too_large":case"page_unreachable":case"browser_session_interrupted":case"site_export_not_found":case"site_export_format_unavailable":case"site_export_read_failed":case"site_export_image_not_found":return"extraction";case"image_url_invalid":case"image_url_not_https":case"image_host_unresolvable":case"image_host_blocked":case"image_redirect_invalid":case"image_source_unavailable":case"image_too_large":case"image_redirect_limit":case"image_base64_invalid":case"image_source_missing":case"image_source_ambiguous":case"image_source_type_invalid":case"image_artifact_not_found":case"image_page_too_large":case"page_image_not_found":case"page_image_candidates_unusable":case"image_format_unsupported":case"image_storage_unconfigured":case"featured_image_unhostable":return"invalid_request";case"insufficient_balance":return"billing";case"vendor_unavailable":case"mcp_http_error":return"service_unavailable";default:return t}}function Fe(e){switch(W(e)){case"request_aborted":return"The request was cancelled before it completed.";case"captcha_exhausted":case"captcha_or_blocked":return"The target site returned a verification challenge. Heavy traffic can temporarily increase these challenges; retry in a few minutes.";case"location_mismatch":return"The target site returned results for a different location. Please retry.";case"proxy_tunnel_failed":case"proxy_unavailable":return"The web request could not establish a working connection. Please retry later.";case"harvest_timeout":return"The request timed out before it completed. Please retry with a smaller request.";case"mcp_request_timeout":return"The MCP caller stopped waiting before a final response arrived. Durable work may still be running; continue with the returned Task or job handle instead of restarting it.";case"concurrency_limit_exceeded":return"The service is available, but this account has reached its parallel-operation limit. Wait for an active operation to finish or add capacity.";case"unauthorized":return"Authentication is required to perform this operation.";case"forbidden":return"This account does not have permission to perform this operation.";case"invalid_request":return"The request is invalid. Check the supplied inputs and try again.";case"rate_limited":return"This account is sending requests too quickly. Wait briefly and retry.";case"upstream_rate_limited":return"The connected provider is temporarily rate-limiting requests. Wait briefly and retry.";case"mcp_http_error":return"The API returned an unexpected HTTP response.";case"insufficient_balance":return"This account does not have enough Credits for the operation.";case"extraction_failed":return"The target could not be extracted. Please retry later.";case"response_lost":return"The connection was lost before the response arrived. The operation may have already completed and been billed on our side \u2014 check credits_info before retrying blind.";case"vendor_unavailable":return"MCP Scraper's internal browser services require MCP Scraper team attention. Servers or IPs are down until this is fixed \u2014 this is not caused by your request. Please retry in a few minutes.";case"page_not_found":return"The page could not be found. It may have been moved or deleted.";case"page_forbidden":return"The site refused the request.";case"page_rate_limited":return"The site is rate-limiting requests right now. Retrying after a short wait usually works.";case"page_server_error":return"The page's own server returned an error. This is on the target site's side, not something a retry here can fix.";case"page_http_error":return"The page returned an unexpected HTTP response.";case"bot_check_unresolved":return"This site has automated-traffic protection (a bot/CAPTCHA check) that could not be resolved in time. Some sites are simply not extractable this way.";case"page_too_large":return"The page is larger than we can safely process.";case"page_unreachable":return"The page did not respond in time, or the connection could not be completed.";case"browser_session_interrupted":return"The browser session closed before extraction completed. MCP Scraper will retry once with a fresh session; retry the request if this persists.";case"site_export_not_found":return"The site export or requested page was not found, expired, or does not belong to this account.";case"site_export_format_unavailable":return"That representation was not retained for this export page. Read the export manifest and request one of the formats marked available.";case"site_export_read_failed":return"The stored site export could not be read or did not pass its integrity check.";case"site_export_image_not_found":return"The requested site-export image was not found, expired, or does not belong to this account.";case"image_url_invalid":return"The image source is not a valid URL. Use a stable public HTTPS URL or inline image bytes.";case"image_url_not_https":return"Commons image sources must use a public HTTPS URL or inline image bytes.";case"image_host_unresolvable":return"The image host could not be resolved. Use a different stable public URL or inline image bytes.";case"image_host_blocked":return"The image source resolves to a network address Commons cannot fetch safely.";case"image_redirect_invalid":return"The image source redirected to an invalid or unsafe destination.";case"image_source_unavailable":return"The image source could not be fetched. Host a different stable public URL or provide inline image bytes.";case"image_too_large":return"The image exceeds the Commons image size limit.";case"image_redirect_limit":return"The image source exceeded the Commons redirect limit.";case"image_base64_invalid":return"The supplied inline image bytes are invalid.";case"image_source_missing":return"Provide exactly one of a public image or webpage URL, inline image bytes, or an owner-scoped image artifact.";case"image_source_ambiguous":return"Provide only one image source: sourceUrl, imageBase64, or artifactId.";case"image_source_type_invalid":return"Use sourceType only with inline image bytes or an owner-scoped image artifact.";case"image_artifact_not_found":return"The image artifact was not found, expired, or does not belong to this account.";case"image_page_too_large":return"The webpage is too large for bounded Commons image discovery.";case"page_image_not_found":return"The webpage did not expose a usable featured or content image.";case"page_image_candidates_unusable":return"The webpage exposed image candidates, but none returned supported public image bytes.";case"image_format_unsupported":return"Commons accepts JPEG, PNG, GIF, or WebP image bytes.";case"image_storage_unconfigured":return"Commons image storage is temporarily unavailable.";case"featured_image_unhostable":return"The required featured image could not be hosted for Commons publication.";default:return xe}}function Pe(e){let t=W(e.errorCode),r=typeof e.retryAfterSeconds=="number"&&Number.isFinite(e.retryAfterSeconds)?Math.max(1,Math.min(86400,Math.trunc(e.retryAfterSeconds))):void 0,s=e.chargeStatus==="not_charged"||e.chargeStatus==="refund_pending"||e.chargeStatus==="refunded"||e.chargeStatus==="charged"?e.chargeStatus:void 0,i=typeof e.errorType=="string"&&Ce.has(e.errorType)?e.errorType:void 0,a=ve(e.details);return{error_code:t,error_type:i??Me(t),message:Fe(t),retryable:e.retryable===!0,...r?{retry_after_seconds:r}:{},...s?{charge_status:s}:{},...a?{details:a}:{}}}function ae(e){return e.replace(/https?:\/\/[^\s]*(?:brightdata|onkernel|kernel\.sh)[^\s]*/gi,"the service").replace(/bright\s*data/gi,"the service").replace(/brightdata/gi,"the service").replace(/kernel\.sh\s+sessions?/gi,"sessions").replace(/kernel\.sh\s+session/gi,"this session").replace(/kernel\.sh/gi,"the service").replace(/kernel\s+sessions?/gi,"sessions").replace(/kernel\s+session/gi,"this session").replace(/\b(?:sub|cus|sess|proxy)_[A-Za-z0-9_-]+\b/g,"[redacted]").replace(/\b(?:KERNEL|BRIGHTDATA|BRIGHT_DATA)_[A-Z0-9_]*API_KEY\b/g,"BROWSER_SERVICE_API_KEY").replace(/\bkernel\b/gi,"the service").replace(/ +/g," ").trim()}var Q=class extends Error{constructor(r){super(`CAPTCHA detected. ${r}`);this.instructions=r}instructions;name="CaptchaError"},Z=class extends Error{constructor(r,s){super(r);this.cause=s}cause;name="ExtractionError"},ee=class extends Error{constructor(r){super(`Google Maps search did not reach a complete result state (${r})`);this.pageState=r}pageState;name="MapsSearchIncompletePageError"},te=class extends Error{name="RequestAbortedError";constructor(t="Request aborted before harvest completed"){super(t)}},re=class extends Error{name="PaaInteractionInterruptedError"},se=class extends Error{name="LocationMismatchError";constructor(t="Google returned results for a different location than requested"){super(t)}};var O=["oauth_store_metadata","oauth_clients","oauth_client_redirect_uris","oauth_codes","oauth_code_exchanges","oauth_refresh_families","oauth_refresh_tokens","oauth_refresh_rotations","oauth_security_events","oauth_rate_limit_buckets"],h=["oauth_clients_inactivity_idx","oauth_clients_cleanup_idx","oauth_codes_expiry_idx","oauth_codes_client_idx","oauth_code_exchanges_replay_expiry_idx","oauth_refresh_families_lifecycle_idx","oauth_refresh_families_user_status_idx","oauth_refresh_families_client_idx","oauth_refresh_families_cleanup_idx","oauth_refresh_tokens_expiry_idx","oauth_refresh_rotations_replay_expiry_idx","oauth_security_events_retention_idx","oauth_rate_limit_buckets_expiry_idx"],oe={oauth_clients:"oauth_clients_legacy_raw",oauth_codes:"oauth_codes_legacy_raw",oauth_tokens:"oauth_tokens_legacy_raw"},b="strftime('%Y-%m-%dT%H:%M:%fZ','now')";async function x(e,t,r=!1){try{await e.execute(t)}catch(s){let i=s.code;if(!r||i!=="SQL_PARSE_ERROR")throw s}}var He=[`CREATE TABLE IF NOT EXISTS oauth_store_metadata (
|
|
1
|
+
import{createClient as Be}from"@libsql/client/http";import{createHash as U,randomBytes as P,randomUUID as g,scryptSync as pe,timingSafeEqual as Ge}from"crypto";import{z as p}from"zod";var D="2026-06-26 21:58:03";var dt="Google returned a CAPTCHA. Run with --headless=false to re-warm the browser profile, then retry.",xe="This operation is temporarily unavailable. Please retry later.",ye=["request_aborted","captcha_exhausted","captcha_or_blocked","location_mismatch","proxy_tunnel_failed","proxy_unavailable","harvest_timeout","mcp_request_timeout","mcp_http_error","extraction_failed","concurrency_limit_exceeded","unauthorized","forbidden","invalid_request","rate_limited","upstream_rate_limited","insufficient_balance","service_unavailable","response_lost","vendor_unavailable","page_not_found","page_forbidden","page_rate_limited","page_server_error","page_http_error","bot_check_unresolved","page_too_large","page_unreachable","browser_session_interrupted","site_export_not_found","site_export_format_unavailable","site_export_read_failed","site_export_image_not_found","image_url_invalid","image_url_not_https","image_host_unresolvable","image_host_blocked","image_redirect_invalid","image_source_unavailable","image_too_large","image_redirect_limit","image_base64_invalid","image_source_missing","image_source_ambiguous","image_source_type_invalid","image_artifact_not_found","image_page_too_large","page_image_not_found","page_image_candidates_unusable","image_format_unsupported","image_storage_unconfigured","featured_image_unhostable"],Ie=["request_aborted","captcha","location_mismatch","connection","timeout","extraction","concurrency_limit","unauthorized","forbidden","invalid_request","rate_limit","billing","service_unavailable"];function w(e){return e?JSON.stringify(e):null}function ne(e){if(e==null)return null;try{let t=typeof e=="string"?JSON.parse(e):e;if(!t||typeof t!="object"||Array.isArray(t))return null;let r=t;if(typeof r.error_code!="string"||!ie.has(r.error_code))return null;let s=Pe({errorCode:r.error_code,errorType:r.error_type,retryable:r.retryable,retryAfterSeconds:r.retry_after_seconds,chargeStatus:r.charge_status,details:r.details});if(typeof r.message=="string"){let i=ae(r.message).trim().slice(0,1e3);i&&!/https?:\/\//i.test(i)&&(s.message=i)}return s}catch{return null}}var ie=new Set(ye),Ce=new Set(Ie),Ue=new Set(["active","limit","operation","upgrade_url","topup_url","upgrade_command","slots_per_pack","pack_price_usd","balance_credits","required_credits","retry_guidance","http_status"]),De=12,ke=500;function W(e){return typeof e=="string"&&ie.has(e)?e:"service_unavailable"}function ve(e){if(!e||typeof e!="object"||Array.isArray(e))return;let t={};for(let[r,s]of Object.entries(e)){if(Object.keys(t).length>=De)break;if(Ue.has(r))if(typeof s=="string"){if((r==="upgrade_url"||r==="topup_url")&&!Xe(s))continue;let i=ae(s).slice(0,ke);i&&(t[r]=i)}else(typeof s=="number"&&Number.isFinite(s)||typeof s=="boolean"||s===null)&&(t[r]=s)}return Object.keys(t).length>0?t:void 0}function Xe(e){if(e.startsWith("/")&&!e.startsWith("//"))return!0;try{let t=new URL(e);return t.protocol==="https:"&&(t.hostname==="mcpscraper.dev"||t.hostname.endsWith(".mcpscraper.dev"))}catch{return!1}}function Me(e){let t=W(e);switch(t){case"captcha_exhausted":case"captcha_or_blocked":return"captcha";case"proxy_tunnel_failed":case"proxy_unavailable":case"response_lost":return"connection";case"harvest_timeout":case"mcp_request_timeout":return"timeout";case"concurrency_limit_exceeded":return"concurrency_limit";case"rate_limited":case"upstream_rate_limited":return"rate_limit";case"extraction_failed":case"page_not_found":case"page_forbidden":case"page_rate_limited":case"page_server_error":case"page_http_error":case"bot_check_unresolved":case"page_too_large":case"page_unreachable":case"browser_session_interrupted":case"site_export_not_found":case"site_export_format_unavailable":case"site_export_read_failed":case"site_export_image_not_found":return"extraction";case"image_url_invalid":case"image_url_not_https":case"image_host_unresolvable":case"image_host_blocked":case"image_redirect_invalid":case"image_source_unavailable":case"image_too_large":case"image_redirect_limit":case"image_base64_invalid":case"image_source_missing":case"image_source_ambiguous":case"image_source_type_invalid":case"image_artifact_not_found":case"image_page_too_large":case"page_image_not_found":case"page_image_candidates_unusable":case"image_format_unsupported":case"image_storage_unconfigured":case"featured_image_unhostable":return"invalid_request";case"insufficient_balance":return"billing";case"vendor_unavailable":case"mcp_http_error":return"service_unavailable";default:return t}}function Fe(e){switch(W(e)){case"request_aborted":return"The request was cancelled before it completed.";case"captcha_exhausted":case"captcha_or_blocked":return"The target site returned a verification challenge. Heavy traffic can temporarily increase these challenges; retry in a few minutes.";case"location_mismatch":return"The target site returned results for a different location. Please retry.";case"proxy_tunnel_failed":case"proxy_unavailable":return"The web request could not establish a working connection. Please retry later.";case"harvest_timeout":return"The request timed out before it completed. Please retry with a smaller request.";case"mcp_request_timeout":return"The MCP caller stopped waiting before a final response arrived. Durable work may still be running; continue with the returned Task or job handle instead of restarting it.";case"concurrency_limit_exceeded":return"The service is available, but this account has reached its parallel-operation limit. Wait for an active operation to finish or add capacity.";case"unauthorized":return"Authentication is required to perform this operation.";case"forbidden":return"This account does not have permission to perform this operation.";case"invalid_request":return"The request is invalid. Check the supplied inputs and try again.";case"rate_limited":return"This account is sending requests too quickly. Wait briefly and retry.";case"upstream_rate_limited":return"The connected provider is temporarily rate-limiting requests. Wait briefly and retry.";case"mcp_http_error":return"The API returned an unexpected HTTP response.";case"insufficient_balance":return"This account does not have enough Credits for the operation.";case"extraction_failed":return"The target could not be extracted. Please retry later.";case"response_lost":return"The connection was lost before the response arrived. The operation may have already completed and been billed on our side \u2014 check credits_info before retrying blind.";case"vendor_unavailable":return"MCP Scraper's internal browser services require MCP Scraper team attention. Servers or IPs are down until this is fixed \u2014 this is not caused by your request. Please retry in a few minutes.";case"page_not_found":return"The page could not be found. It may have been moved or deleted.";case"page_forbidden":return"The site refused the request.";case"page_rate_limited":return"The site is rate-limiting requests right now. Retrying after a short wait usually works.";case"page_server_error":return"The page's own server returned an error. This is on the target site's side, not something a retry here can fix.";case"page_http_error":return"The page returned an unexpected HTTP response.";case"bot_check_unresolved":return"This site has automated-traffic protection (a bot/CAPTCHA check) that could not be resolved in time. Some sites are simply not extractable this way.";case"page_too_large":return"The page is larger than we can safely process.";case"page_unreachable":return"The page did not respond in time, or the connection could not be completed.";case"browser_session_interrupted":return"The browser session closed before extraction completed. MCP Scraper will retry once with a fresh session; retry the request if this persists.";case"site_export_not_found":return"The site export or requested page was not found, expired, or does not belong to this account.";case"site_export_format_unavailable":return"That representation was not retained for this export page. Read the export manifest and request one of the formats marked available.";case"site_export_read_failed":return"The stored site export could not be read or did not pass its integrity check.";case"site_export_image_not_found":return"The requested site-export image was not found, expired, or does not belong to this account.";case"image_url_invalid":return"The image source is not a valid URL. Use a stable public HTTPS URL or inline image bytes.";case"image_url_not_https":return"Commons image sources must use a public HTTPS URL or inline image bytes.";case"image_host_unresolvable":return"The image host could not be resolved. Use a different stable public URL or inline image bytes.";case"image_host_blocked":return"The image source resolves to a network address Commons cannot fetch safely.";case"image_redirect_invalid":return"The image source redirected to an invalid or unsafe destination.";case"image_source_unavailable":return"The image source could not be fetched. Host a different stable public URL or provide inline image bytes.";case"image_too_large":return"The image exceeds the Commons image size limit.";case"image_redirect_limit":return"The image source exceeded the Commons redirect limit.";case"image_base64_invalid":return"The supplied inline image bytes are invalid.";case"image_source_missing":return"Provide exactly one of a public image or webpage URL, inline image bytes, or an owner-scoped image artifact.";case"image_source_ambiguous":return"Provide only one image source: sourceUrl, imageBase64, or artifactId.";case"image_source_type_invalid":return"Use sourceType only with inline image bytes or an owner-scoped image artifact.";case"image_artifact_not_found":return"The image artifact was not found, expired, or does not belong to this account.";case"image_page_too_large":return"The webpage is too large for bounded Commons image discovery.";case"page_image_not_found":return"The webpage did not expose a usable featured or content image.";case"page_image_candidates_unusable":return"The webpage exposed image candidates, but none returned supported public image bytes.";case"image_format_unsupported":return"Commons accepts JPEG, PNG, GIF, or WebP image bytes.";case"image_storage_unconfigured":return"Commons image storage is temporarily unavailable.";case"featured_image_unhostable":return"The required featured image could not be hosted for Commons publication.";default:return xe}}function Pe(e){let t=W(e.errorCode),r=typeof e.retryAfterSeconds=="number"&&Number.isFinite(e.retryAfterSeconds)?Math.max(1,Math.min(86400,Math.trunc(e.retryAfterSeconds))):void 0,s=e.chargeStatus==="not_charged"||e.chargeStatus==="refund_pending"||e.chargeStatus==="refunded"||e.chargeStatus==="charged"?e.chargeStatus:void 0,i=typeof e.errorType=="string"&&Ce.has(e.errorType)?e.errorType:void 0,a=ve(e.details);return{error_code:t,error_type:i??Me(t),message:Fe(t),retryable:e.retryable===!0,...r?{retry_after_seconds:r}:{},...s?{charge_status:s}:{},...a?{details:a}:{}}}function ae(e){return e.replace((new RegExp("https?:\\/\\/[^\\s]*(?:\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061|on\u006b\u0065\u0072\u006e\u0065\u006c|\u006b\u0065\u0072\u006e\u0065\u006c\\.sh)[^\\s]*", "gi")),"the service").replace(/bright\s*data/gi,"the service").replace((new RegExp("\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061", "gi")),"the service").replace((new RegExp("\u006b\u0065\u0072\u006e\u0065\u006c\\.sh\\s+sessions?", "gi")),"sessions").replace((new RegExp("\u006b\u0065\u0072\u006e\u0065\u006c\\.sh\\s+session", "gi")),"this session").replace((new RegExp("\u006b\u0065\u0072\u006e\u0065\u006c\\.sh", "gi")),"the service").replace((new RegExp("\u006b\u0065\u0072\u006e\u0065\u006c\\s+sessions?", "gi")),"sessions").replace((new RegExp("\u006b\u0065\u0072\u006e\u0065\u006c\\s+session", "gi")),"this session").replace(/\b(?:sub|cus|sess|proxy)_[A-Za-z0-9_-]+\b/g,"[redacted]").replace((new RegExp("\\b(?:\u004b\u0045\u0052\u004e\u0045\u004c|\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041|\u0042\u0052\u0049\u0047\u0048\u0054_\u0044\u0041\u0054\u0041)_[A-Z0-9_]*API_KEY\\b", "g")),"BROWSER_SERVICE_API_KEY").replace((new RegExp("\\b\u006b\u0065\u0072\u006e\u0065\u006c\\b", "gi")),"the service").replace(/ +/g," ").trim()}var Q=class extends Error{constructor(r){super(`CAPTCHA detected. ${r}`);this.instructions=r}instructions;name="CaptchaError"},Z=class extends Error{constructor(r,s){super(r);this.cause=s}cause;name="ExtractionError"},ee=class extends Error{constructor(r){super(`Google Maps search did not reach a complete result state (${r})`);this.pageState=r}pageState;name="MapsSearchIncompletePageError"},te=class extends Error{name="RequestAbortedError";constructor(t="Request aborted before harvest completed"){super(t)}},re=class extends Error{name="PaaInteractionInterruptedError"},se=class extends Error{name="LocationMismatchError";constructor(t="Google returned results for a different location than requested"){super(t)}};var O=["oauth_store_metadata","oauth_clients","oauth_client_redirect_uris","oauth_codes","oauth_code_exchanges","oauth_refresh_families","oauth_refresh_tokens","oauth_refresh_rotations","oauth_security_events","oauth_rate_limit_buckets"],h=["oauth_clients_inactivity_idx","oauth_clients_cleanup_idx","oauth_codes_expiry_idx","oauth_codes_client_idx","oauth_code_exchanges_replay_expiry_idx","oauth_refresh_families_lifecycle_idx","oauth_refresh_families_user_status_idx","oauth_refresh_families_client_idx","oauth_refresh_families_cleanup_idx","oauth_refresh_tokens_expiry_idx","oauth_refresh_rotations_replay_expiry_idx","oauth_security_events_retention_idx","oauth_rate_limit_buckets_expiry_idx"],oe={oauth_clients:"oauth_clients_legacy_raw",oauth_codes:"oauth_codes_legacy_raw",oauth_tokens:"oauth_tokens_legacy_raw"},b="strftime('%Y-%m-%dT%H:%M:%fZ','now')";async function x(e,t,r=!1){try{await e.execute(t)}catch(s){let i=s.code;if(!r||i!=="SQL_PARSE_ERROR")throw s}}var He=[`CREATE TABLE IF NOT EXISTS oauth_store_metadata (
|
|
2
2
|
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
3
3
|
store_id TEXT NOT NULL UNIQUE,
|
|
4
4
|
schema_version INTEGER NOT NULL CHECK (schema_version > 0),
|
|
@@ -188,14 +188,14 @@ import{createClient as Be}from"@libsql/client/http";import{createHash as U,rando
|
|
|
188
188
|
max_questions INTEGER NOT NULL,
|
|
189
189
|
status TEXT NOT NULL,
|
|
190
190
|
outcome TEXT,
|
|
191
|
-
|
|
191
|
+
\u006b\u0065\u0072\u006e\u0065\u006c_session_id TEXT,
|
|
192
192
|
question_count INTEGER,
|
|
193
193
|
duration_ms INTEGER,
|
|
194
194
|
error TEXT,
|
|
195
195
|
will_retry INTEGER,
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
196
|
+
\u006b\u0065\u0072\u006e\u0065\u006c_delete_started INTEGER NOT NULL DEFAULT 0,
|
|
197
|
+
\u006b\u0065\u0072\u006e\u0065\u006c_delete_succeeded INTEGER,
|
|
198
|
+
\u006b\u0065\u0072\u006e\u0065\u006c_delete_error TEXT,
|
|
199
199
|
browser_close_succeeded INTEGER,
|
|
200
200
|
browser_close_error TEXT,
|
|
201
201
|
debug_json TEXT,
|
|
@@ -767,8 +767,8 @@ import{createClient as Be}from"@libsql/client/http";import{createHash as U,rando
|
|
|
767
767
|
id TEXT PRIMARY KEY,
|
|
768
768
|
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
769
769
|
name TEXT NOT NULL,
|
|
770
|
-
|
|
771
|
-
|
|
770
|
+
\u006b\u0065\u0072\u006e\u0065\u006c_profile_name TEXT NOT NULL,
|
|
771
|
+
\u006b\u0065\u0072\u006e\u0065\u006c_proxy_id TEXT NOT NULL,
|
|
772
772
|
proxy_type TEXT NOT NULL DEFAULT 'isp',
|
|
773
773
|
country TEXT NOT NULL DEFAULT 'US',
|
|
774
774
|
status TEXT NOT NULL DEFAULT 'ready',
|
|
@@ -827,7 +827,7 @@ import{createClient as Be}from"@libsql/client/http";import{createHash as U,rando
|
|
|
827
827
|
ON CONFLICT(scope, key_hash, window_start)
|
|
828
828
|
DO UPDATE SET count = count + 1, updated_at = datetime('now')
|
|
829
829
|
RETURNING count
|
|
830
|
-
`,args:[e,u,a]}),_=Number(o.rows[0]?.count);if(!Number.isSafeInteger(_)||_<1)throw new Error("Rate-limit bucket mutation did not return a valid count");return{allowed:_<=r,count:_,remaining:Math.max(0,r-_),resetSeconds:Math.max(1,a+s-i)}}async function Kt(e){let t=P(32).toString("hex"),r=U("sha256").update(t).digest("hex"),s=new Date(Date.now()+3600*1e3).toISOString();return await n().execute({sql:"INSERT INTO password_reset_tokens (token, user_id, expires_at) VALUES (?, ?, ?)",args:[r,e,s]}),t}async function Vt(e){let t=n(),r=U("sha256").update(e).digest("hex"),i=(await t.execute({sql:"SELECT user_id, expires_at, used FROM password_reset_tokens WHERE token = ?",args:[r]})).rows[0];return!i||Number(i.used)||new Date(String(i.expires_at))<new Date?null:(await t.execute({sql:"UPDATE password_reset_tokens SET used = 1 WHERE token = ?",args:[r]}),Number(i.user_id))}async function zt(e){let t=me();return await n().execute({sql:"UPDATE users SET api_key = ?, key_active = 1 WHERE id = ?",args:[t,e]}),t}async function Qt(e){await n().execute({sql:"UPDATE users SET key_active = 0 WHERE id = ?",args:[e]})}async function Zt(){return(await n().execute("SELECT id, email, name, created_at, active FROM users ORDER BY created_at DESC")).rows.map(t=>L({...t,api_key:"",key_active:1}))}async function er(e){await n().execute({sql:"UPDATE users SET active = 0 WHERE id = ?",args:[e]})}function S(e){return{id:String(e.id),user_id:Number(e.user_id),status:String(e.status),query:String(e.query),options:String(e.options),callback_url:e.callback_url!=null?String(e.callback_url):null,result:e.result!=null?String(e.result):null,error:e.error!=null?String(e.error):null,public_error_json:e.public_error_json!=null?String(e.public_error_json):null,created_at:String(e.created_at),started_at:e.started_at!=null?String(e.started_at):null,completed_at:e.completed_at!=null?String(e.completed_at):null}}function y(e){return e==null?null:Number(e)===1}function F(e){if(e==null)return null;try{return JSON.parse(String(e))}catch{return null}}function Ne(e){return{id:String(e.id),job_id:String(e.job_id),user_id:Number(e.user_id),attempt_number:Number(e.attempt_number),max_attempts:Number(e.max_attempts),query:String(e.query),location:e.location!=null?String(e.location):null,max_questions:Number(e.max_questions),status:String(e.status),outcome:e.outcome!=null?String(e.outcome):null
|
|
830
|
+
`,args:[e,u,a]}),_=Number(o.rows[0]?.count);if(!Number.isSafeInteger(_)||_<1)throw new Error("Rate-limit bucket mutation did not return a valid count");return{allowed:_<=r,count:_,remaining:Math.max(0,r-_),resetSeconds:Math.max(1,a+s-i)}}async function Kt(e){let t=P(32).toString("hex"),r=U("sha256").update(t).digest("hex"),s=new Date(Date.now()+3600*1e3).toISOString();return await n().execute({sql:"INSERT INTO password_reset_tokens (token, user_id, expires_at) VALUES (?, ?, ?)",args:[r,e,s]}),t}async function Vt(e){let t=n(),r=U("sha256").update(e).digest("hex"),i=(await t.execute({sql:"SELECT user_id, expires_at, used FROM password_reset_tokens WHERE token = ?",args:[r]})).rows[0];return!i||Number(i.used)||new Date(String(i.expires_at))<new Date?null:(await t.execute({sql:"UPDATE password_reset_tokens SET used = 1 WHERE token = ?",args:[r]}),Number(i.user_id))}async function zt(e){let t=me();return await n().execute({sql:"UPDATE users SET api_key = ?, key_active = 1 WHERE id = ?",args:[t,e]}),t}async function Qt(e){await n().execute({sql:"UPDATE users SET key_active = 0 WHERE id = ?",args:[e]})}async function Zt(){return(await n().execute("SELECT id, email, name, created_at, active FROM users ORDER BY created_at DESC")).rows.map(t=>L({...t,api_key:"",key_active:1}))}async function er(e){await n().execute({sql:"UPDATE users SET active = 0 WHERE id = ?",args:[e]})}function S(e){return{id:String(e.id),user_id:Number(e.user_id),status:String(e.status),query:String(e.query),options:String(e.options),callback_url:e.callback_url!=null?String(e.callback_url):null,result:e.result!=null?String(e.result):null,error:e.error!=null?String(e.error):null,public_error_json:e.public_error_json!=null?String(e.public_error_json):null,created_at:String(e.created_at),started_at:e.started_at!=null?String(e.started_at):null,completed_at:e.completed_at!=null?String(e.completed_at):null}}function y(e){return e==null?null:Number(e)===1}function F(e){if(e==null)return null;try{return JSON.parse(String(e))}catch{return null}}function Ne(e){return{id:String(e.id),job_id:String(e.job_id),user_id:Number(e.user_id),attempt_number:Number(e.attempt_number),max_attempts:Number(e.max_attempts),query:String(e.query),location:e.location!=null?String(e.location):null,max_questions:Number(e.max_questions),status:String(e.status),outcome:e.outcome!=null?String(e.outcome):null,\u006b\u0065\u0072\u006e\u0065\u006c_session_id:e.\u006b\u0065\u0072\u006e\u0065\u006c_session_id!=null?String(e.\u006b\u0065\u0072\u006e\u0065\u006c_session_id):null,question_count:e.question_count!=null?Number(e.question_count):null,duration_ms:e.duration_ms!=null?Number(e.duration_ms):null,error:e.error!=null?String(e.error):null,will_retry:y(e.will_retry),\u006b\u0065\u0072\u006e\u0065\u006c_delete_started:Number(e.\u006b\u0065\u0072\u006e\u0065\u006c_delete_started??0)===1,\u006b\u0065\u0072\u006e\u0065\u006c_delete_succeeded:y(e.\u006b\u0065\u0072\u006e\u0065\u006c_delete_succeeded),\u006b\u0065\u0072\u006e\u0065\u006c_delete_error:e.\u006b\u0065\u0072\u006e\u0065\u006c_delete_error!=null?String(e.\u006b\u0065\u0072\u006e\u0065\u006c_delete_error):null,browser_close_succeeded:y(e.browser_close_succeeded),browser_close_error:e.browser_close_error!=null?String(e.browser_close_error):null,browser_provider:e.browser_provider!=null?String(e.browser_provider):null,provider_session_id:e.provider_session_id!=null?String(e.provider_session_id):null,provider_session_observed_at:e.provider_session_observed_at!=null?String(e.provider_session_observed_at):null,provider_lookup_status:e.provider_lookup_status!=null?String(e.provider_lookup_status):null,provider_status:e.provider_status!=null?String(e.provider_status):null,provider_error_code:e.provider_error_code!=null?String(e.provider_error_code):null,provider_error_message:e.provider_error_message!=null?String(e.provider_error_message):null,provider_reason_source:e.provider_reason_source!=null?String(e.provider_reason_source):null,provider_duration_ms:e.provider_duration_ms!=null?Number(e.provider_duration_ms):null,provider_bandwidth_bytes:e.provider_bandwidth_bytes!=null?Number(e.provider_bandwidth_bytes):null,provider_captcha:y(e.provider_captcha),provider_checked_at:e.provider_checked_at!=null?String(e.provider_checked_at):null,provider_detail:F(e.provider_detail_json),provider_disconnect_observed:y(e.provider_disconnect_observed),provider_disconnect_message:e.provider_disconnect_message!=null?String(e.provider_disconnect_message):null,debug:F(e.debug_json),created_at:String(e.created_at),completed_at:e.completed_at!=null?String(e.completed_at):null}}function f(e){let{public_error_json:t,...r}=e;return{...r,options:JSON.parse(e.options),result:e.result?JSON.parse(e.result):null,publicError:ne(t)}}async function tr(e,t,r,s){let i=g();return await ze(i,e,t,r,s),i}async function ze(e,t,r,s,i){if(!/^[a-zA-Z0-9-]{1,100}$/.test(e))throw new Error("job id is invalid");await n().execute({sql:"INSERT INTO jobs (id, user_id, query, options, callback_url) VALUES (?, ?, ?, ?, ?)",args:[e,t,r,JSON.stringify(s),i??null]})}async function rr(e,t,r){let s=g();return await n().execute({sql:"INSERT INTO jobs (id, user_id, query, options, status, started_at) VALUES (?, ?, ?, ?, 'running', datetime('now'))",args:[s,e,t,JSON.stringify(r)]}),s}async function sr(e,t,r,s){let i=await n().execute({sql:`INSERT INTO jobs (id, user_id, query, options, status, started_at)
|
|
831
831
|
VALUES (?, ?, ?, ?, 'running', datetime('now'))
|
|
832
832
|
ON CONFLICT(id) DO NOTHING`,args:[e,t,r,JSON.stringify(s)]}),a=await G(e,t);if(!a)throw new Error(`getOrCreateRunningJob: row missing immediately after insert/lookup for ${e}`);return{job:a,created:i.rowsAffected===1}}async function nr(e){await n().execute({sql:`
|
|
833
833
|
INSERT INTO harvest_attempts (
|
|
@@ -862,14 +862,14 @@ import{createClient as Be}from"@libsql/client/http";import{createHash as U,rando
|
|
|
862
862
|
UPDATE harvest_attempts SET
|
|
863
863
|
status = 'finished',
|
|
864
864
|
outcome = ?,
|
|
865
|
-
|
|
865
|
+
\u006b\u0065\u0072\u006e\u0065\u006c_session_id = ?,
|
|
866
866
|
question_count = ?,
|
|
867
867
|
duration_ms = ?,
|
|
868
868
|
error = ?,
|
|
869
869
|
will_retry = ?,
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
870
|
+
\u006b\u0065\u0072\u006e\u0065\u006c_delete_started = ?,
|
|
871
|
+
\u006b\u0065\u0072\u006e\u0065\u006c_delete_succeeded = ?,
|
|
872
|
+
\u006b\u0065\u0072\u006e\u0065\u006c_delete_error = ?,
|
|
873
873
|
browser_close_succeeded = ?,
|
|
874
874
|
browser_close_error = ?,
|
|
875
875
|
browser_provider = COALESCE(browser_provider, ?),
|
|
@@ -901,7 +901,7 @@ import{createClient as Be}from"@libsql/client/http";import{createHash as U,rando
|
|
|
901
901
|
debug_json = ?,
|
|
902
902
|
completed_at = ?
|
|
903
903
|
WHERE job_id = ? AND attempt_number = ?
|
|
904
|
-
`,args:[e.outcome,e
|
|
904
|
+
`,args:[e.outcome,e.\u006b\u0065\u0072\u006e\u0065\u006cSessionId,e.questionCount,e.durationMs,e.error,e.willRetry?1:0,e.\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted?1:0,e.\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded==null?null:e.\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded?1:0,e.\u006b\u0065\u0072\u006e\u0065\u006cDeleteError,e.browserCloseSucceeded==null?null:e.browserCloseSucceeded?1:0,e.browserCloseError,e.browserProvider??null,e.providerSessionId??null,e.providerLookupStatus??null,e.providerStatus??null,e.providerErrorCode??null,e.providerErrorMessage??null,e.providerReasonSource??null,e.providerDurationMs??null,e.providerBandwidthBytes??null,e.providerCaptcha==null?null:e.providerCaptcha?1:0,e.providerCheckedAt??null,e.providerDetail==null?null:be(e.providerDetail),e.providerDisconnectObserved==null?null:e.providerDisconnectObserved?1:0,e.providerDisconnectMessage??null,e.debug==null?null:JSON.stringify(e.debug),e.completedAt,e.jobId,e.attemptNumber]})}function be(e,t=4096){let r=JSON.stringify(e);return r.length<=t?r:JSON.stringify({truncated:!0})}async function ur(e){return(await n().execute({sql:`
|
|
905
905
|
UPDATE harvest_attempts SET
|
|
906
906
|
provider_lookup_status = ?,
|
|
907
907
|
provider_status = ?,
|
|
@@ -916,12 +916,12 @@ import{createClient as Be}from"@libsql/client/http";import{createHash as U,rando
|
|
|
916
916
|
WHERE id = ?
|
|
917
917
|
AND job_id = ?
|
|
918
918
|
AND user_id = ?
|
|
919
|
-
AND browser_provider = '
|
|
919
|
+
AND browser_provider = '\u0062\u0072\u0069\u0067\u0068\u0074_\u0064\u0061\u0074\u0061_browser_api'
|
|
920
920
|
AND provider_session_id = ?
|
|
921
921
|
AND COALESCE(provider_lookup_status, '') <> 'complete'
|
|
922
922
|
`,args:[e.lookupStatus,e.providerStatus??null,e.errorCode??null,e.errorMessage??null,e.reasonSource??null,e.durationMs??null,e.bandwidthBytes??null,e.captcha==null?null:e.captcha?1:0,e.checkedAt,e.detail==null?null:be(e.detail),e.attemptId,e.jobId,e.userId,e.providerSessionId]})).rowsAffected===1}async function cr(e=25){let t=Math.max(1,Math.min(100,Math.trunc(e)));return(await n().execute({sql:`
|
|
923
923
|
SELECT * FROM harvest_attempts
|
|
924
|
-
WHERE browser_provider = '
|
|
924
|
+
WHERE browser_provider = '\u0062\u0072\u0069\u0067\u0068\u0074_\u0064\u0061\u0074\u0061_browser_api'
|
|
925
925
|
AND provider_session_id IS NOT NULL
|
|
926
926
|
AND provider_lookup_status IN ('pending', 'temporary_error')
|
|
927
927
|
ORDER BY COALESCE(provider_checked_at, created_at) ASC
|
package/dist/chunk-DKQZNSYM.js
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import{t as _}from"./chunk-7BEWBRMT.js";import{AsyncLocalStorage as N}from"async_hooks";var C="MCP_SCRAPER_ALLOW_HEADLESS_HARVEST_TEST",d=new N;function D(e,t){return d.run(e,t)}function l(){return d.getStore()}import{randomUUID as T}from"crypto";var p=166667e-10,L=.0001333336,h=new Set(["serp","fb_search","fb_ad","instagram"]),A=.00111,R=4,g=4e-4;function n(e,t){let r=process.env[e]?.trim();if(!r)return t;let o=Number(r);return Number.isFinite(o)&&o>=0?o:t}var m=n("NANGO_USD_PER_CONNECTION_MONTH",1),U=n("NANGO_USD_PER_FUNCTION_RUN",1e-4),O=n("NANGO_USD_PER_PROXY_REQUEST",1e-4),S=n("NANGO_USD_PER_COMPUTE_SEC",2e-4),b=n("
|
|
1
|
+
import{t as _}from"./chunk-7BEWBRMT.js";import{AsyncLocalStorage as N}from"async_hooks";var C="MCP_SCRAPER_ALLOW_HEADLESS_HARVEST_TEST",d=new N;function D(e,t){return d.run(e,t)}function l(){return d.getStore()}import{randomUUID as T}from"crypto";var p=166667e-10,L=.0001333336,h=new Set(["serp","fb_search","fb_ad","instagram"]),A=.00111,R=4,g=4e-4;function n(e,t){let r=process.env[e]?.trim();if(!r)return t;let o=Number(r);return Number.isFinite(o)&&o>=0?o:t}var m=n("NANGO_USD_PER_CONNECTION_MONTH",1),U=n("NANGO_USD_PER_FUNCTION_RUN",1e-4),O=n("NANGO_USD_PER_PROXY_REQUEST",1e-4),S=n("NANGO_USD_PER_COMPUTE_SEC",2e-4),b=n("\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041_BROWSER_USD_PER_GB",8);function u(e,t){return Math.max(0,e)/1e3*(t?L:p)}function i(e,t){return e==="fal_wizper"?Math.max(0,t)/R*A:e==="deepinfra_qwen"?Math.max(0,t)/1e3*g:e==="openrouter"||e==="mcp_memory_video"||e==="mcp_memory_ai"?Math.max(0,t):e==="nango_connection"?Math.max(0,t)*m:e==="nango_function_run"?Math.max(0,t)*U:e==="nango_proxy_request"?Math.max(0,t)*O:e==="nango_compute"?Math.max(0,t)*S:e==="\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061_browser_api"?Math.max(0,t)/1e9*b:0}var E=!1,s=null;async function c(){if(!E)return s||(s=x().finally(()=>{s=null}),s)}async function x(){let e=_(),t=await e.execute(`
|
|
2
2
|
SELECT
|
|
3
|
-
(SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('
|
|
4
|
-
AND (SELECT COUNT(*) FROM pragma_table_info('
|
|
3
|
+
(SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('\u006b\u0065\u0072\u006e\u0065\u006c_session_log', 'vendor_usage_log', 'cost_probe_runs')) = 3
|
|
4
|
+
AND (SELECT COUNT(*) FROM pragma_table_info('\u006b\u0065\u0072\u006e\u0065\u006c_session_log') WHERE name IN ('proxy_source', 'proxy_type', 'method')) = 3
|
|
5
5
|
AND (SELECT COUNT(*) FROM pragma_table_info('vendor_usage_log') WHERE name IN ('method', 'source_key', 'provider_duration_ms', 'provider_captcha', 'provider_status')) = 5
|
|
6
6
|
AND (SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'vendor_usage_log_vendor_source_key') = 1
|
|
7
7
|
AND (SELECT COUNT(*) FROM pragma_table_info('cost_probe_runs') WHERE name IN ('units', 'unit_type', 'mode')) = 3
|
|
8
8
|
AS ready
|
|
9
9
|
`);if(Number(t.rows[0]?.ready??0)===1){E=!0;return}await e.execute(`
|
|
10
|
-
CREATE TABLE IF NOT EXISTS
|
|
10
|
+
CREATE TABLE IF NOT EXISTS \u006b\u0065\u0072\u006e\u0065\u006c_session_log (
|
|
11
11
|
id TEXT PRIMARY KEY,
|
|
12
|
-
|
|
12
|
+
\u006b\u0065\u0072\u006e\u0065\u006c_session_id TEXT,
|
|
13
13
|
op TEXT,
|
|
14
14
|
source TEXT NOT NULL,
|
|
15
15
|
probe_run_id TEXT,
|
|
@@ -26,7 +26,7 @@ import{t as _}from"./chunk-7BEWBRMT.js";import{AsyncLocalStorage as N}from"async
|
|
|
26
26
|
error TEXT,
|
|
27
27
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
28
28
|
)
|
|
29
|
-
`),await e.execute("CREATE INDEX IF NOT EXISTS
|
|
29
|
+
`),await e.execute("CREATE INDEX IF NOT EXISTS \u006b\u0065\u0072\u006e\u0065\u006c_session_log_op ON \u006b\u0065\u0072\u006e\u0065\u006c_session_log(op)"),await e.execute("CREATE INDEX IF NOT EXISTS \u006b\u0065\u0072\u006e\u0065\u006c_session_log_probe ON \u006b\u0065\u0072\u006e\u0065\u006c_session_log(probe_run_id)"),await e.execute("CREATE INDEX IF NOT EXISTS \u006b\u0065\u0072\u006e\u0065\u006c_session_log_created ON \u006b\u0065\u0072\u006e\u0065\u006c_session_log(created_at)");try{await e.execute("ALTER TABLE \u006b\u0065\u0072\u006e\u0065\u006c_session_log ADD COLUMN proxy_source TEXT")}catch{}try{await e.execute("ALTER TABLE \u006b\u0065\u0072\u006e\u0065\u006c_session_log ADD COLUMN proxy_type TEXT")}catch{}try{await e.execute("ALTER TABLE \u006b\u0065\u0072\u006e\u0065\u006c_session_log ADD COLUMN method TEXT")}catch{}await e.execute(`
|
|
30
30
|
CREATE TABLE IF NOT EXISTS vendor_usage_log (
|
|
31
31
|
id TEXT PRIMARY KEY,
|
|
32
32
|
op TEXT,
|
|
@@ -54,20 +54,20 @@ import{t as _}from"./chunk-7BEWBRMT.js";import{AsyncLocalStorage as N}from"async
|
|
|
54
54
|
success INTEGER NOT NULL DEFAULT 0,
|
|
55
55
|
error TEXT,
|
|
56
56
|
http_only INTEGER NOT NULL DEFAULT 0,
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
57
|
+
\u006b\u0065\u0072\u006e\u0065\u006c_used INTEGER NOT NULL DEFAULT 0,
|
|
58
|
+
\u006b\u0065\u0072\u006e\u0065\u006c_sessions INTEGER NOT NULL DEFAULT 0,
|
|
59
|
+
\u006b\u0065\u0072\u006e\u0065\u006c_seconds_total REAL NOT NULL DEFAULT 0,
|
|
60
|
+
\u006b\u0065\u0072\u006e\u0065\u006c_tier_observed TEXT,
|
|
61
|
+
est_\u006b\u0065\u0072\u006e\u0065\u006c_cost_usd_headless REAL NOT NULL DEFAULT 0,
|
|
62
|
+
est_\u006b\u0065\u0072\u006e\u0065\u006c_cost_usd_headful REAL NOT NULL DEFAULT 0,
|
|
63
63
|
vendor_cost_usd REAL NOT NULL DEFAULT 0,
|
|
64
64
|
charged_credits REAL NOT NULL DEFAULT 0,
|
|
65
65
|
margin_usd_headless_starter REAL NOT NULL DEFAULT 0,
|
|
66
66
|
margin_usd_headful_starter REAL NOT NULL DEFAULT 0,
|
|
67
67
|
notes TEXT
|
|
68
68
|
)
|
|
69
|
-
`),await e.execute("CREATE INDEX IF NOT EXISTS cost_probe_runs_tool ON cost_probe_runs(tool)");try{await e.execute("ALTER TABLE cost_probe_runs ADD COLUMN units REAL")}catch{}try{await e.execute("ALTER TABLE cost_probe_runs ADD COLUMN unit_type TEXT")}catch{}try{await e.execute("ALTER TABLE cost_probe_runs ADD COLUMN mode TEXT")}catch{}E=!0}async function w(e){try{await c();let t=l(),r=Math.max(0,e.closedAtMs-e.openedAtMs);await _().execute({sql:`INSERT INTO
|
|
70
|
-
(id,
|
|
71
|
-
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,args:[T(),e
|
|
69
|
+
`),await e.execute("CREATE INDEX IF NOT EXISTS cost_probe_runs_tool ON cost_probe_runs(tool)");try{await e.execute("ALTER TABLE cost_probe_runs ADD COLUMN units REAL")}catch{}try{await e.execute("ALTER TABLE cost_probe_runs ADD COLUMN unit_type TEXT")}catch{}try{await e.execute("ALTER TABLE cost_probe_runs ADD COLUMN mode TEXT")}catch{}E=!0}async function w(e){try{await c();let t=l(),r=Math.max(0,e.closedAtMs-e.openedAtMs);await _().execute({sql:`INSERT INTO \u006b\u0065\u0072\u006e\u0065\u006c_session_log
|
|
70
|
+
(id, \u006b\u0065\u0072\u006e\u0065\u006c_session_id, op, source, probe_run_id, user_id, stealth, headless_sent, proxy_used, proxy_source, proxy_type, fallback, opened_at, closed_at, duration_ms, est_cost_usd_headless, est_cost_usd_headful, error, method)
|
|
71
|
+
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,args:[T(),e.\u006b\u0065\u0072\u006e\u0065\u006cSessionId??null,t?.op??null,e.source,t?.probeRunId??null,t?.userId??null,a(e.stealth),a(e.headlessSent),a(e.proxyUsed),e.proxySource??null,e.proxyType??null,e.fallback?1:0,new Date(e.openedAtMs).toISOString(),new Date(e.closedAtMs).toISOString(),r,u(r,!1),u(r,!0),e.error??null,t?.subOp??null]})}catch(t){console.warn("[cost-telemetry] record\u004b\u0065\u0072\u006e\u0065\u006cSession failed:",t instanceof Error?t.message:String(t))}}async function F(e){try{await c();let t=l();return(await _().execute({sql:`INSERT OR IGNORE INTO vendor_usage_log
|
|
72
72
|
(id, op, probe_run_id, user_id, vendor, model, units, unit_type, est_cost_usd, error, method, source_key, provider_duration_ms, provider_captcha, provider_status)
|
|
73
73
|
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,args:[T(),e.op??t?.op??null,e.probeRunId??t?.probeRunId??null,e.userId??t?.userId??null,e.vendor,e.model??null,e.units,e.unitType,i(e.vendor,e.units),e.error??null,e.method??t?.subOp??null,e.sourceKey??null,e.providerDurationMs??null,a(e.providerCaptcha),e.providerStatus??null]})).rowsAffected===1}catch(t){return console.warn("[cost-telemetry] recordVendorUsage failed:",t instanceof Error?t.message:String(t)),!1}}function a(e){return e==null?null:e?1:0}export{C as a,D as b,l as c,h as d,i as e,c as f,w as g,F as h};
|