mcp-scraper 0.88.2 → 0.89.0

package/dist/chunk-CCYWSJNG.js

@@ -0,0 +1,16 @@
+var s={message:"Discover public Facebook Reel URLs, retain partial inventories, and transcribe selected videos. Facebook extraction now dismisses optional login dialogs and retries failed public acquisition."};var _={reset:"\x1B[0m",cyan:"\x1B[36m",lime:"\x1B[32m",amber:"\x1B[33m",red:"\x1B[31m",muted:"\x1B[90m",bold:"\x1B[1m"};function r(t,e,n){return n?`${_[e]}${t}${_.reset}`:t}function o(t,e,n){let a=t.padEnd(9," ");return`  ${r(a,"muted",n)} ${e.join(r("  .  ","muted",n))}`}function p(t){let e=t.color??!0,n=t.apiKeyConfigured?"$MCP_SCRAPER_API_KEY":"sk_live_your_key",a=String.raw`
+ __  __  ____ ____ 
+|  \/  |/ ___|  _ \
+| |\/| | |   | |_) |
+| |  | | |___|  __/
+|_|  |_|\____|_|
+
+ ____   ____ ____      _    ____  _____ ____
+/ ___| / ___|  _ \    / \  |  _ \| ____|  _ \
+\___ \| |   | |_) |  / _ \ | |_) |  _| | |_) |
+ ___) | |___|  _ <  / ___ \|  __/| |___|  _ <
+|____/ \____|_| \_\/_/   \_\_|   |_____|_| \_\
+`,i=[`MCP_SCRAPER_API_KEY=${n} npx -y -p mcp-scraper@latest \\`,"  mcp-scraper-cli agent install claude --apply"].join(`
+`),c=["[mcp_servers.mcp-scraper]",'command = "npx"','args = ["-y", "-p", "mcp-scraper@latest", "mcp-scraper"]',`env = { MCP_SCRAPER_API_KEY = "${n}" }`].join(`
+`);return[r(`mcp-scraper v${t.version}`,"bold",e),r("> mcp-scraper-install","muted",e),r(a,"amber",e),`${r("MCP Scraper Agent","cyan",e)}  .  v${t.version}  .  mcpscraper.dev`,"1/1 install surfaces ready",r(`Newest in v${t.version}: ${s.message}`,"lime",e),"",`${r("Tools","cyan",e)}  ${r("(376 MCP tools)","muted",e)}`,o("search",["harvest_paa","search_serp","maps_search","maps_place_intel"],e),o("extract",["extract_url","map_site_urls","extract_site","audit_site","directory_workflow"],e),o("build",["create_editorial_reading_room","rank_tracker_workflow","portable HTML"],e),o("media",["youtube_harvest","youtube_transcribe","facebook_reels_inventory","facebook_ad_search","facebook_page_intel","facebook_ad_transcribe","facebook_video_transcribe","instagram_profile_content","instagram_media_download","reddit_thread"],e),o("browser",["serp_identity_create","serp_identity_list","browser_open","browser_profile_connect","browser_profile_list","browser_close","browser_screenshot","browser_read","browser_locate","browser_replay_mark","browser_replay_annotate"],e),o("connect",["list_service_connections","describe_service_connection_tool","import_service_connection_to_memory","export_connected_service_data","renew_connected_data_download","read_service_connection","call_service_connection_action"],e),o("commons",["commons_search_entities","commons_get_entity_linkset","commons_prepare_entity","commons_submit_entity","commons_prepare_publication","commons_claim_publication","commons_publish_editorial","commons_get_publication"],e),o("account",["credits_info","reports","MCP resources"],e),o("memory",["memory-put","memory-get","memory-search","list-vaults","record-fact","list-scheduled-actions"],e),`${r("Workflows","cyan",e)}  ${r("(MCP + CLI + API)","muted",e)}`,o("route",["workflow_list","workflow_suggest","workflow_run","workflow_step","workflow_status","workflow_artifact_read"],e),o("seo",["directory","agent-packet","competitive audit","map/serp comparison","PAA/AIO briefs","scheduled runs"],e),"",r("Usage tips:","amber",e),"Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.","Run mcp-scraper in a human terminal to print this card; MCP clients get the same command as a silent stdio server.","Explicit card command: npx -y -p mcp-scraper@latest mcp-scraper-install","Hosted browser sessions use direct/no-proxy egress by default.","Customer auth setup: run browser_profile_connect, send the watch_url, let the user sign in, then call browser_profile_list until AUTHENTICATED.","Connected account ranges: call export_connected_service_data once. It handles Gmail, Calendar, Zoom, and Resend pagination; do not loop read_service_connection over individual records.","Connected account RAG: call import_service_connection_to_memory for one bounded approved read. It writes a redacted, untrusted snapshot to a stable Memory path and embeds it for search.","Stack logins / reconnect: run browser_profile_connect again with the same profile name and another domain to add accounts or refresh a login.","Start with workflow_suggest for broad jobs like market analysis, ICP research, CRO audits, brand briefs, content gaps, and AI visibility.","For MCP clients, use mcp-scraper so one install can mix SERP, Maps, browser, reports, and saved MCP resources.","If you hit the concurrency limit, add 2 browsers for $5/month with mcp-scraper-cli billing concurrency checkout.","",`${r("Ready.","lime",e)} Install the combined MCP server with one command:`,"",r("Setup doctor","amber",e),"npx -y -p mcp-scraper@latest mcp-scraper-cli doctor","",r("Hosted profile setup","amber",e),'In your MCP client, call browser_profile_connect with email="seo@example.com" and domain="chatgpt.com".',"Give the returned watch_url to the user. After they sign in, call browser_profile_list, then browser_open with the returned profile. Add more logins by calling browser_profile_connect again with the same profile and a new domain.","",r("Claude Code one-command setup","amber",e),i,"Then fully exit Claude Code and open a new Claude terminal. Check with: claude mcp list","",r("Codex config","amber",e),c,"",r("Claude Desktop Extension","amber",e),"Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb","",r("Safety note:","muted",e),"mcp-scraper prints this card only when stdin/stdout are an interactive TTY. In MCP clients it writes only JSON-RPC to stdout.","Use --stdio or MCP_SCRAPER_FORCE_STDIO=1 to force server mode from a terminal.",""].join(`
+`)}export{p as a};

package/dist/chunk-CFI6CXIV.js

@@ -0,0 +1,182 @@
+import{a as j,b as T,c as P,d as C,f as S}from"./chunk-TMB56NCA.js";import{W as w}from"./chunk-MZN4U5BL.js";import{Ib as D,d as m,e as I,i as u,s,zb as N}from"./chunk-GMWKPIYX.js";import{createHash as W}from"crypto";import{gunzipSync as $,gzipSync as q}from"zlib";var y="site-extracts/",L=10080*60*1e3,H=900*1e3;function U(){return process.env.SITE_EXTRACT_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim()||null}function v(){return process.env.VERCEL==="1"||process.env.NODE_ENV==="production"}function p(){return{prefix:y,artifactTtlMs:L,downloadTtlMs:H,token:U()}}function h(t){return j(t,y)}async function V(t){let e=await P({policy:p(),ownerId:t.ownerId,artifactKey:`${t.jobId}.zip`,createdAt:t.createdAt,filename:`${t.jobId}-site-export.zip`,contentType:"application/zip",content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"bundle"}}async function G(t){let e=await T({policy:p(),ownerId:t.ownerId,scopeSegments:[t.jobId,"images"],artifactKey:`${t.imageId}.bin`,createdAt:new Date,filename:t.filename,contentType:t.contentType,content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"image",imageId:t.imageId,sourceUrl:t.sourceUrl,sourcePage:t.sourcePage}}async function O(t){return T({policy:p(),ownerId:t.ownerId,scopeSegments:[t.jobId,"content"],artifactKey:`${t.chunkKey}.json.gz`,createdAt:t.createdAt,filename:`${t.chunkKey}.json.gz`,contentType:"application/gzip",content:t.content})}async function Q(t){return C({policy:p(),artifactId:t.artifactId,ownerId:t.ownerId})}async function J(t){return S({policy:p(),artifactId:t,maxBytes:50*1024*1024})}async function Z(t){return h(t.artifactId)!==t.ownerId?null:J(t.artifactId)}async function k(t){return h(t.artifactId)!==t.ownerId?null:S({policy:p(),artifactId:t.artifactId,maxBytes:25*1024*1024})}async function tt(t){return h(t.artifactId)!==t.ownerId?null:S({policy:p(),artifactId:t.artifactId,maxBytes:10*1024*1024})}async function et(t={}){let e=t.now??new Date,n=U();if(!n)return{deleted:0,store:v()?"none":"local"};if(!t.force&&!(e.getUTCHours()===3&&e.getUTCMinutes()===21))return{deleted:0,store:"private-vercel-blob",skipped:!0};let r=e.getTime()-L,{list:i,del:a}=await import("@vercel/blob"),o,c=0;for(let d=0;d<20;d+=1){let l=await i({prefix:y,token:n,limit:1e3,cursor:o}),f=l.blobs.filter(A=>new Date(A.uploadedAt).getTime()<=r);if(f.length>0&&(await a(f.map(A=>A.pathname),{token:n}),c+=f.length),!l.hasMore||!l.cursor)break;o=l.cursor}return{deleted:c,store:"private-vercel-blob"}}var F=10,b=20*1024*1024;function E(t){return W("sha256").update(t).digest("hex")}function B(t){return!t.acquiredHtml||t.extractionStatus==="failed"?null:{pageId:t.pageId??E(t.archivedUrl??t.url),url:t.archivedUrl??t.url,html:t.acquiredHtml,mainHtml:t.mainHtml??"",markdown:t.bodyMarkdown,htmlSha256:t.htmlSha256??E(t.acquiredHtml),markdownSha256:t.markdownSha256??E(t.bodyMarkdown)}}function R(t){return Buffer.from(JSON.stringify({version:"site-extract-content.v1",pages:t}))}async function M(t){let e=t.pages.flatMap(o=>{let c=B(o);return c?[c]:[]}),n=new Map,r=[],i=0,a=async()=>{if(!r.length)return;let o=R(r);if(o.length>b)throw new Error(`site export content chunk exceeds ${b} bytes`);let c=E(r.map(l=>l.pageId).join("\0")).slice(0,16),d=await O({ownerId:t.ownerId,jobId:t.jobId,createdAt:t.createdAt,chunkKey:`content-${String(i).padStart(4,"0")}-${c}`,content:q(o,{level:6})});for(let l of r)n.set(l.url,{artifactId:d.artifactId,artifactSha256:d.sha256,pageId:l.pageId,uncompressedBytes:o.length});i++,r=[]};for(let o of e){let c=[...r,o];if(r.length>0&&(c.length>F||R(c).length>b)&&await a(),r.push(o),R(r).length>b)throw new Error(`page ${o.pageId} exceeds the durable content chunk limit`)}return await a(),n}function it(t){let e=new Map;return async n=>{let r=e.get(n.artifactId);if(!r){let a=await k({artifactId:n.artifactId,ownerId:t});if(!a)throw new Error("site export content chunk is missing or unauthorized");if(E(a)!==n.artifactSha256)throw new Error("site export content chunk checksum mismatch");let o=$(a);if(o.length>b)throw new Error("site export content chunk exceeds its read limit");if(r=JSON.parse(o.toString("utf8")),r.version!=="site-extract-content.v1"||!Array.isArray(r.pages))throw new Error("site export content chunk has an unsupported contract");for(e.set(n.artifactId,r);e.size>2;)e.delete(e.keys().next().value)}let i=r.pages.find(a=>a.pageId===n.pageId);if(!i)throw new Error("site export page is missing from its content chunk");if(E(i.html)!==i.htmlSha256||E(i.markdown)!==i.markdownSha256)throw new Error("site export page checksum mismatch");return i}}var g="xray_entitlement";function pt(t){let e=Math.max(1,Number(t.options.effectiveMaxPages??t.options.maxPages??1)),n=Math.max(e,Number(t.options.requestedMaxPages??e)),r=n>e;return{requestedMaxPages:n,effectiveMaxPages:e,creditLimited:r,creditTruncated:r&&t.totalUrls>=e}}function Et(t,e=!1){return t.successfulUrls===0?"failed":t.failedUrls>0||t.remainingUrls>0||e?"partial":"complete"}function _(t){let e=Number(t.total_urls??0),n=t.attempted_urls==null,r=Number(n?t.done_urls??0:t.attempted_urls),i=Number(n?t.done_urls??0:t.successful_urls??0),a=Number(t.failed_urls??Math.max(0,r-i));return{id:String(t.id),userId:t.user_id!=null?Number(t.user_id):null,idempotencyKey:t.idempotency_key!=null?String(t.idempotency_key):null,requestFingerprint:t.request_fingerprint!=null?String(t.request_fingerprint):null,status:t.status!=null?String(t.status):"pending",startUrl:String(t.start_url??""),options:t.options?JSON.parse(String(t.options)):{},totalUrls:e,doneUrls:r,attemptedUrls:r,successfulUrls:i,failedUrls:a,remainingUrls:Math.max(0,e-r),artifacts:t.artifacts?JSON.parse(String(t.artifacts)):null,error:t.error!=null?String(t.error):null,publicError:I(t.public_error_json),billedMc:t.billed_mc!=null?Number(t.billed_mc):null,createdAt:String(t.created_at??""),updatedAt:String(t.updated_at??"")}}async function mt(t,e,n,r){await s().execute({sql:`INSERT INTO site_extract_jobs (id, user_id, status, start_url, options, created_at, updated_at)
+          VALUES (?, ?, 'pending', ?, ?, datetime('now'), datetime('now'))`,args:[t,e,n,JSON.stringify(r)]})}async function K(t,e){let r=await s().execute({sql:"SELECT * FROM site_extract_jobs WHERE user_id = ? AND idempotency_key = ? LIMIT 1",args:[t,e]});return r.rows[0]?_(r.rows[0]):null}async function gt(t){let n=await s().execute({sql:`INSERT OR IGNORE INTO site_extract_jobs
+            (id, user_id, status, start_url, options, idempotency_key, request_fingerprint, created_at, updated_at)
+          VALUES (?, ?, 'pending', ?, ?, ?, ?, datetime('now'), datetime('now'))`,args:[t.jobId,t.userId,t.startUrl,JSON.stringify(t.options),t.idempotencyKey,t.requestFingerprint]}),r=await K(t.userId,t.idempotencyKey);if(!r)throw new Error("idempotent extract job was not persisted");return{job:r,created:n.rowsAffected===1,conflict:r.requestFingerprint!==t.requestFingerprint}}async function x(t){let n=await s().execute({sql:"SELECT * FROM site_extract_jobs WHERE id = ?",args:[t]});return n.rows[0]?_(n.rows[0]):null}async function _t(t){return(await s().execute({sql:"SELECT * FROM site_extract_jobs WHERE user_id = ? ORDER BY created_at DESC LIMIT 50",args:[t]})).rows.map(r=>_(r))}async function xt(t=25){return(await s().execute({sql:`SELECT * FROM site_extract_jobs
+          WHERE billed_mc IS NULL AND status IN ('complete', 'partial', 'failed')
+            AND (next_settlement_at IS NULL OR next_settlement_at <= datetime('now'))
+          ORDER BY updated_at ASC LIMIT ?`,args:[Math.max(1,Math.min(100,Math.round(t)))]})).rows.map(r=>_(r))}async function ft(t=25,e=2){let n=s(),r=Math.max(1,Math.min(60,Math.round(e)));return(await n.execute({sql:`SELECT j.* FROM site_extract_jobs j
+          LEFT JOIN billing_debits d
+            ON d.idempotency_key = json_extract(j.options, '$.debitKey')
+           AND d.user_id = j.user_id
+           AND d.status = 'applied'
+          WHERE j.status = 'pending'
+            AND j.billed_mc IS NULL
+            AND (
+              d.idempotency_key IS NOT NULL
+              OR (
+                json_extract(j.options, '$.billingClass') = ?
+                AND json_extract(j.options, '$.heldMc') = 0
+                AND json_extract(j.options, '$.debitKey') IS NULL
+                AND json_extract(j.options, '$.xraySetupReceipt.billingClass') = ?
+                AND json_extract(j.options, '$.xraySetupReceipt.consumesMcpScraperCredits') = 0
+              )
+            )
+            AND (j.next_dispatch_at IS NULL OR j.next_dispatch_at <= datetime('now'))
+            AND j.updated_at <= datetime('now', ?)
+          ORDER BY j.updated_at ASC LIMIT ?`,args:[g,g,`-${r} minutes`,Math.max(1,Math.min(100,Math.round(t)))]})).rows.map(a=>_(a))}async function bt(t){return await s().execute({sql:`UPDATE site_extract_jobs
+          SET dispatch_attempts = dispatch_attempts + 1,
+              updated_at = datetime('now'),
+              next_dispatch_at = datetime('now', '+5 minutes'),
+              dispatch_error = NULL,
+              status = CASE
+                WHEN dispatch_attempts + 1 >= 10 OR created_at <= datetime('now', '-6 hours') THEN 'failed'
+                ELSE status
+              END,
+              error = CASE
+                WHEN dispatch_attempts + 1 >= 10 OR created_at <= datetime('now', '-6 hours')
+                  THEN 'Background crawl was acknowledged repeatedly but never started; the hold will be refunded.'
+                ELSE error
+              END
+          WHERE id = ? AND status = 'pending'`,args:[t]}),x(t)}async function wt(t,e){let n=s(),r=u(e).slice(0,1e3);return await n.execute({sql:`UPDATE site_extract_jobs SET
+            dispatch_attempts = dispatch_attempts + 1,
+            dispatch_error = ?,
+            next_dispatch_at = datetime('now', '+5 minutes'),
+            updated_at = datetime('now'),
+            status = CASE
+              WHEN dispatch_attempts + 1 >= 10 OR created_at <= datetime('now', '-6 hours') THEN 'failed'
+              ELSE status
+            END,
+            error = CASE
+              WHEN dispatch_attempts + 1 >= 10 OR created_at <= datetime('now', '-6 hours')
+                THEN 'Background crawl could not be dispatched after repeated attempts; the hold will be refunded.'
+              ELSE error
+            END
+          WHERE id = ? AND status = 'pending'`,args:[r,t]}),x(t)}async function St(t,e){await s().execute({sql:`UPDATE site_extract_jobs SET
+            settlement_attempts = settlement_attempts + 1,
+            settlement_error = ?,
+            next_settlement_at = datetime('now', '+15 minutes'),
+            updated_at = datetime('now')
+          WHERE id = ? AND billed_mc IS NULL`,args:[u(e).slice(0,1e3),t]})}async function At(t=25,e=60){let n=s(),r=Math.max(15,Math.min(1440,Math.round(e)));return(await n.execute({sql:`SELECT j.* FROM site_extract_jobs j
+          LEFT JOIN billing_debits d
+            ON d.idempotency_key = json_extract(j.options, '$.debitKey')
+           AND d.user_id = j.user_id
+           AND d.status = 'applied'
+          WHERE j.status = 'running'
+            AND j.billed_mc IS NULL
+            AND (
+              json_extract(j.options, '$.debitKey') IS NULL
+              OR d.idempotency_key IS NOT NULL
+            )
+            AND j.updated_at <= datetime('now', ?)
+          ORDER BY j.updated_at ASC LIMIT ?`,args:[`-${r} minutes`,Math.max(1,Math.min(100,Math.round(t)))]})).rows.map(a=>_(a))}async function Nt(t,e=60){let n=s(),r=Math.max(15,Math.min(1440,Math.round(e)));return(await n.execute({sql:`UPDATE site_extract_jobs
+          SET status = 'failed',
+              error = 'Background crawl stopped without a heartbeat; completed pages were preserved and settlement is pending.',
+              public_error_json = ?,
+              updated_at = datetime('now')
+          WHERE id = ?
+            AND status = 'running'
+            AND billed_mc IS NULL
+            AND updated_at <= datetime('now', ?)`,args:[m({error_code:"extraction_failed",error_type:"extraction",message:"The background crawl stopped without a heartbeat. Completed pages were preserved; retry after settlement completes.",retryable:!0,charge_status:"refund_pending"}),t,`-${r} minutes`]})).rowsAffected===1}async function Tt(t){return(await s().execute({sql:`UPDATE site_extract_jobs
+          SET status = 'running', updated_at = datetime('now')
+          WHERE id = ? AND status = 'failed'`,args:[t]})).rowsAffected!==1?null:x(t)}async function yt(t,e){await s().execute({sql:`UPDATE site_extract_jobs SET billed_mc = 0, settlement_error = ?, updated_at = datetime('now')
+          WHERE id = ? AND billed_mc IS NULL`,args:[u(e).slice(0,1e3),t]})}async function ht(t,e){await s().execute({sql:`UPDATE site_extract_jobs
+            SET status = 'running', total_urls = MAX(total_urls, ?), updated_at = datetime('now')
+          WHERE id = ? AND status IN ('pending', 'running')`,args:[e,t]})}async function Rt(t,e){if(e.length===0)return;let n=s(),r=await x(t),i=r?.userId!=null?await M({ownerId:String(r.userId),jobId:t,createdAt:r.createdAt,pages:e}):new Map;await n.batch([...e.map(a=>{let{discoveryLinks:o,acquiredHtml:c,mainHtml:d,...l}=a;return l.contentRef=i.get(a.archivedUrl??a.url)??a.contentRef,{sql:`INSERT INTO site_extract_pages (job_id, url, page)
+            SELECT ?, ?, ?
+            WHERE EXISTS (
+              SELECT 1 FROM site_extract_jobs
+              WHERE id = ? AND status IN ('pending', 'running')
+            )
+            ON CONFLICT(job_id, url) DO UPDATE SET page = excluded.page
+            WHERE NOT CASE
+                    WHEN json_valid(site_extract_pages.page) = 0 THEN 0
+                    WHEN json_extract(site_extract_pages.page, '$.extractionStatus') IS NOT NULL
+                      THEN json_extract(site_extract_pages.page, '$.extractionStatus') = 'successful'
+                    ELSE COALESCE(
+                      json_extract(site_extract_pages.page, '$.status') >= 200
+                      AND json_extract(site_extract_pages.page, '$.status') < 300
+                      AND json_extract(site_extract_pages.page, '$.wordCount') > 0,
+                      0
+                    )
+                  END
+               OR CASE
+                    WHEN json_valid(excluded.page) = 0 THEN 0
+                    WHEN json_extract(excluded.page, '$.extractionStatus') IS NOT NULL
+                      THEN json_extract(excluded.page, '$.extractionStatus') = 'successful'
+                    ELSE COALESCE(
+                      json_extract(excluded.page, '$.status') >= 200
+                      AND json_extract(excluded.page, '$.status') < 300
+                      AND json_extract(excluded.page, '$.wordCount') > 0,
+                      0
+                    )
+                  END`,args:[t,a.archivedUrl??a.url,JSON.stringify(l),t]}}),{sql:`UPDATE site_extract_jobs SET
+            done_urls = (SELECT COUNT(*) FROM site_extract_pages WHERE job_id = ?),
+            attempted_urls = (SELECT COUNT(*) FROM site_extract_pages WHERE job_id = ?),
+            successful_urls = (
+              SELECT COUNT(*) FROM site_extract_pages
+              WHERE job_id = ? AND CASE
+                WHEN json_extract(page, '$.extractionStatus') IS NOT NULL
+                  THEN json_extract(page, '$.extractionStatus') = 'successful'
+                ELSE json_extract(page, '$.status') >= 200
+                  AND json_extract(page, '$.status') < 300
+                  AND json_extract(page, '$.wordCount') > 0
+              END
+            ),
+            failed_urls = (
+              SELECT COUNT(*) FROM site_extract_pages
+              WHERE job_id = ? AND NOT CASE
+                WHEN json_extract(page, '$.extractionStatus') IS NOT NULL
+                  THEN json_extract(page, '$.extractionStatus') = 'successful'
+                ELSE COALESCE(
+                  json_extract(page, '$.status') >= 200
+                  AND json_extract(page, '$.status') < 300
+                  AND json_extract(page, '$.wordCount') > 0,
+                  0
+                )
+              END
+            ),
+            updated_at = datetime('now')
+          WHERE id = ? AND status IN ('pending', 'running')`,args:[t,t,t,t,t]}],"write")}async function It(t){return(await s().execute({sql:"SELECT page FROM site_extract_pages WHERE job_id = ? ORDER BY rowid",args:[t]})).rows.map(r=>JSON.parse(String(r.page)))}async function Dt(t){return(await s().execute({sql:"SELECT page FROM site_extract_pages WHERE job_id = ?",args:[t]})).rows.map(r=>JSON.parse(String(r.page)))}async function jt(t,e=181){let n=s(),r=Math.max(1,Math.min(5001,Math.round(e)));return(await n.execute({sql:`SELECT DISTINCT CAST(images.value AS TEXT) AS url
+          FROM site_extract_pages p,
+               json_each(CASE WHEN json_valid(p.page) THEN p.page ELSE '{}' END, '$.imageLinks') images
+          WHERE p.job_id = ?
+            AND images.type = 'text'
+            AND length(CAST(images.value AS TEXT)) BETWEEN 1 AND 4096
+          LIMIT ?`,args:[t,r]})).rows.map(a=>String(a.url))}async function Pt(t){let n=await s().execute({sql:`SELECT COUNT(*) AS n FROM site_extract_pages
+            WHERE job_id = ? AND CASE
+              WHEN json_extract(page, '$.extractionStatus') IS NOT NULL
+                THEN json_extract(page, '$.extractionStatus') = 'successful'
+              ELSE COALESCE(
+                json_extract(page, '$.status') >= 200
+                AND json_extract(page, '$.status') < 300
+                AND json_extract(page, '$.wordCount') > 0,
+                0
+              )
+            END`,args:[t]});return Number(n.rows[0]?.n??0)}async function Ct(t){let n=await s().execute({sql:"SELECT url FROM site_extract_pages WHERE job_id = ?",args:[t]});return new Set(n.rows.map(r=>String(r.url)))}async function z(t,e,n,r=null,i=!1,a=null){return(await s().execute({sql:`UPDATE site_extract_jobs
+            SET status = ?, artifacts = ?, error = ?, public_error_json = ?, updated_at = datetime('now')
+          WHERE id = ? AND (status IN ('pending', 'running') OR (? = 1 AND status = 'failed'))`,args:[n,JSON.stringify(e??[]),r?u(r).slice(0,2e3):null,m(a),t,i?1:0]})).rowsAffected===1}async function Lt(t,e){await z(t,e,"complete")}async function Ut(t,e,n){await s().execute({sql:`UPDATE site_extract_jobs
+            SET status = 'failed', error = ?, public_error_json = ?, updated_at = datetime('now')
+          WHERE id = ? AND status IN ('pending', 'running')`,args:[u(e).slice(0,2e3),m(n),t]})}async function Ot(t,e,n){await s().execute({sql:`UPDATE site_extract_jobs
+            SET status = 'failed', billed_mc = 0, error = ?, public_error_json = ?, updated_at = datetime('now')
+          WHERE id = ? AND status = 'pending' AND billed_mc IS NULL`,args:[u(e).slice(0,2e3),m(n),t]})}async function kt(t,e){let r=(await x(t))?.options.billingClass===g&&e?{...e,charge_status:"not_charged"}:e;await s().execute({sql:`UPDATE site_extract_jobs SET public_error_json = ?, updated_at = datetime('now')
+          WHERE id = ? AND status IN ('complete', 'partial', 'failed')`,args:[m(r),t]})}async function Mt(t,e,n,r,i){let a=s(),o=await x(t);if(!o||o.billedMc!=null)return;if(o.options.billingClass===g){if(!await X(t))throw new Error("xray_extract_zero_charge_finalization_rejected");return}let c=typeof o.options.debitKey=="string"?o.options.debitKey:null;if(c){let f=await D(e,c,Math.max(0,r),w.EXTRACT_SITE_REFUND,i,"extract_site");await a.execute({sql:"UPDATE site_extract_jobs SET billed_mc = ?, updated_at = datetime('now') WHERE id = ? AND billed_mc IS NULL",args:[f.final_amount_mc,t]});return}let d=Math.max(0,n),l=Math.max(0,r);await a.batch([{sql:"UPDATE site_extract_jobs SET billed_mc = -1 WHERE id = ? AND billed_mc IS NULL",args:[t]},{sql:`INSERT INTO credit_lots (user_id, amount_mc, remaining_mc, source, expires_at)
+            SELECT ?, ?, ?, ?, datetime(j.created_at, ?)
+            FROM site_extract_jobs j
+            WHERE j.id = ?
+              AND j.billed_mc = -1
+              AND ? > 0
+              AND changes() = 1
+              AND datetime(j.created_at, ?) > datetime('now')`,args:[e,d,d,w.EXTRACT_SITE_REFUND,N,t,d,N]},{sql:`UPDATE users SET balance_mc = balance_mc + ?
+            WHERE id = ? AND ? > 0 AND changes() = 1`,args:[d,e,d]},{sql:`INSERT INTO ledger (user_id, amount_mc, operation, description)
+            SELECT ?, ?, ?, ? WHERE ? > 0 AND changes() = 1`,args:[e,d,w.EXTRACT_SITE_REFUND,i,d]},{sql:`UPDATE site_extract_jobs SET billed_mc = ?, updated_at = datetime('now')
+            WHERE id = ? AND billed_mc = -1`,args:[l,t]}],"write")}async function X(t){return(await s().execute({sql:`UPDATE site_extract_jobs
+          SET billed_mc = 0, updated_at = datetime('now')
+          WHERE id = ?
+            AND billed_mc IS NULL
+            AND json_extract(options, '$.billingClass') = ?
+            AND json_extract(options, '$.heldMc') = 0
+            AND json_extract(options, '$.debitKey') IS NULL
+            AND json_extract(options, '$.xraySetupReceipt.billingClass') = ?
+            AND json_extract(options, '$.xraySetupReceipt.consumesMcpScraperCredits') = 0
+            AND json_extract(options, '$.xraySetupReceipt.heldMc') = 0
+            AND json_extract(options, '$.xraySetupReceipt.billedMc') = 0`,args:[t,g,g]})).rowsAffected===1}export{y as a,V as b,G as c,Q as d,Z as e,tt as f,et as g,it as h,g as i,pt as j,Et as k,mt as l,K as m,gt as n,x as o,_t as p,xt as q,ft as r,bt as s,wt as t,St as u,At as v,Nt as w,Tt as x,yt as y,ht as z,Rt as A,It as B,Dt as C,jt as D,Pt as E,Ct as F,z as G,Lt as H,Ut as I,Ot as J,kt as K,Mt as L,X as M};

package/dist/chunk-E2WRWV3A.js

@@ -0,0 +1 @@
+import{a as m}from"./chunk-YQZGZBB4.js";import{b as n,ba as p,ca as d,ea as b,g as o,h as i,j as l,m as u,o as c}from"./chunk-GMWKPIYX.js";function f(e){return e instanceof Error?e.message:String(e)}function y(e,t){return e instanceof DOMException&&(e.name==="TimeoutError"||e.name==="AbortError")?!0:/timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(t)}function g(e){return/captcha|recaptcha|unusual traffic|google\.com\/sorry|blocked/i.test(e)}function S(e){return/ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(e)}function E(e){return/proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(e)}function h(e){return/(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\/\/\S*dashboard/i.test(e)}function v(e){return/browser (?:has been )?closed|context (?:has been )?closed|target page, context or browser has been closed|session closed|page has been closed/i.test(e)}function x(e){let t=f(e);return e instanceof u?{error_code:"request_aborted",error_type:"request_aborted",message:o("request_aborted"),retryable:!0,httpStatus:408,terminalStatus:"cancelled"}:e instanceof l||g(t)?{error_code:"captcha_exhausted",error_type:"captcha",message:o("captcha_exhausted"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:m(e,t)?{error_code:"vendor_unavailable",error_type:"service_unavailable",message:o("vendor_unavailable"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:e instanceof c?{error_code:"location_mismatch",error_type:"location_mismatch",message:o("location_mismatch"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:S(t)?{error_code:"proxy_tunnel_failed",error_type:"connection",message:o("proxy_tunnel_failed"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:E(t)?{error_code:"proxy_unavailable",error_type:"connection",message:o("proxy_unavailable"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:v(t)?{error_code:"browser_session_interrupted",error_type:"extraction",message:o("browser_session_interrupted"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:y(e,t)?{error_code:"harvest_timeout",error_type:"timeout",message:o("harvest_timeout"),retryable:!0,httpStatus:504,terminalStatus:"failed"}:h(t)?{error_code:"service_unavailable",error_type:"service_unavailable",message:n,retryable:!0,httpStatus:503,terminalStatus:"failed"}:{error_code:"extraction_failed",error_type:"extraction",message:n,retryable:!1,httpStatus:500,terminalStatus:"failed"}}function w(e){return JSON.stringify({error_code:e.error_code,error_type:e.error_type,message:e.message,retryable:e.retryable})}function _(e,t={}){let r=i({errorCode:e.error_code,errorType:e.error_type,retryable:e.retryable,retryAfterSeconds:t.retryAfterSeconds,chargeStatus:t.chargeStatus,details:t.details});return{error:r.message,...r}}function A(e,t={}){let{error:r,...a}=_(e,t);return a}function C(e,t={}){let r=x(e);return{body:_(r,t),status:r.httpStatus}}function D(e,t){return async r=>{if(r.type==="started"){await p({jobId:e,userId:t,attemptNumber:r.attemptNumber,maxAttempts:r.maxAttempts,query:r.query,location:r.location,maxQuestions:r.maxQuestions,startedAt:r.startedAt});return}if(r.type==="provider_session_started"){await d({jobId:e,attemptNumber:r.attemptNumber,browserProvider:r.provider,providerSessionId:r.providerSessionId,observedAt:r.observedAt});return}await b({jobId:e,attemptNumber:r.attemptNumber,outcome:r.outcome,\u006b\u0065\u0072\u006e\u0065\u006cSessionId:r.\u006b\u0065\u0072\u006e\u0065\u006cSessionId,questionCount:r.questionCount,durationMs:r.durationMs,error:r.error,willRetry:r.willRetry,\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted:r.cleanup.\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted,\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded:r.cleanup.\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded,\u006b\u0065\u0072\u006e\u0065\u006cDeleteError:r.cleanup.\u006b\u0065\u0072\u006e\u0065\u006cDeleteError,browserCloseSucceeded:r.cleanup.browserCloseSucceeded,browserCloseError:r.cleanup.browserCloseError,browserProvider:r.cleanup.provider??null,providerSessionId:r.cleanup.providerSessionId??null,providerLookupStatus:r.cleanup.providerSessionId?"pending":"not_applicable",providerErrorCode:r.cleanup.providerDisconnectObserved?"browser_session_interrupted":null,providerErrorMessage:r.cleanup.providerDisconnectMessage??null,providerReasonSource:r.cleanup.providerDisconnectObserved?"client_runtime":null,providerDisconnectObserved:r.cleanup.providerDisconnectObserved??!1,providerDisconnectMessage:r.cleanup.providerDisconnectMessage??null,debug:r.debug,completedAt:r.completedAt})}}async function I(e,t,r=3){for(let a=1;a<=r;a++){try{let s=await fetch(e,{method:"POST",headers:{"content-type":"application/json"},body:JSON.stringify(t),signal:AbortSignal.timeout(1e4)});if(s.ok)return;console.warn(`[webhook] attempt ${a} \u2192 ${s.status} from ${e}`)}catch(s){console.warn(`[webhook] attempt ${a} failed:`,s instanceof Error?s.message:s)}a<r&&await new Promise(s=>setTimeout(s,1e3*a*2))}console.error(`[webhook] gave up after ${r} attempts for ${e}`)}export{x as a,w as b,_ as c,A as d,C as e,D as f,I as g};