mcp-scraper 0.96.2 → 0.97.0

package/dist/chunk-RYX4FJVH.js

@@ -1 +0,0 @@
-import{c as j}from"./chunk-BPKTUCV7.js";import{k as X}from"./chunk-ZTMQEDIB.js";import{c as A,d as $,g as b,h as C,n as K,o as U,p as G,q as M,r as D,s as W,y as q,z as F}from"./chunk-7WPCGRD7.js";import{a as B,b as L,c as H}from"./chunk-QER2ELZP.js";import{b as k,c as S,d as y,e as x,f as O,g as I,h,i as m,k as E,o as N}from"./chunk-HUNSEGUN.js";import{h as v,i as z}from"./chunk-C7MGJUTU.js";import{a as d,b as J,c as T,d as R}from"./chunk-HNNYXMSI.js";import{randomUUID as oe}from"crypto";import{createHash as Q,randomUUID as V}from"crypto";function _(e,i=new Set){if(e===null||typeof e=="string"||typeof e=="boolean"||typeof e=="number"&&Number.isFinite(e))return e;if(typeof e!="object"||!e)throw new d("Operation inputs must be finite JSON data");if(i.has(e))throw new d("Operation inputs cannot be cyclic");if(!Array.isArray(e)&&![Object.prototype,null].includes(Object.getPrototypeOf(e)))throw new d("Operation inputs must be plain JSON data");i.add(e);try{return Array.isArray(e)?e.map(t=>_(t,i)):Object.fromEntries(Object.keys(e).sort().map(t=>[t,_(e[t],i)]))}finally{i.delete(e)}}async function P(e){if(!Number.isSafeInteger(e.userId)||e.userId<=0)throw new d("Authenticated operation principal is required");if(e.jobId!=null&&(!e.jobId.trim()||e.jobId.length>512))throw new d("Operation job ID is invalid");let i=e.tool.trim();if(!i||i.length>100)throw new d("Operation tool is required and bounded");if(e.idempotencyKey!=null&&(!e.idempotencyKey.trim()||e.idempotencyKey.length>512))throw new d("Operation idempotency key is invalid");let t=e.leaseMs??3e5;if(!Number.isSafeInteger(t)||t<1e3||t>36e5)throw new d("Execution lease must be between one second and one hour");let n=JSON.stringify(_(e.normalizedInput));if(Buffer.byteLength(n)>64*1024)throw new d("Operation inputs exceed 64 KiB");let r=e.idempotencyKey?.trim()??V();return E({userId:e.userId,ownerScope:`user:${e.userId}`,tool:i,jobId:e.jobId??null,normalizedFlags:e.normalizedInput,idempotencyKey:r,requestFingerprint:Q("sha256").update(n).digest("hex")},V(),t,e.storeGate)}async function Y(e){let i=JSON.stringify(_(e.normalizedInput));if(!Number.isSafeInteger(e.userId)||e.userId<=0||Buffer.byteLength(i)>64*1024)throw new d("Invalid operation receipt principal/input");let{readOperationIntentResult:t}=await import("./repository-4EC5EXVN.js"),{operationIdentityHash:n}=await import("./contracts-GUXGFCVV.js");return t(e.userId,n(`user:${e.userId}\0${e.tool}\0${e.idempotencyKey}`),Q("sha256").update(i).digest("hex"))}async function we(e){let i=Date.now()+27e4,t=e.idempotencyKey?.trim()??oe();if(!t||t.length>512)return{status:400,body:{error_code:"invalid_request",message:"Idempotency-Key must contain 1 to 512 characters"}};let n=(a,c,p)=>({...a,body:{...a.body,operation:{runId:c,idempotencyKey:t,state:p}}}),r=(a,c,p)=>n({status:c==="pending"?202:503,headers:{"Retry-After":"2"},body:{...v({errorCode:"response_lost",retryable:!0,chargeStatus:p,details:{retry_guidance:"Retry the identical request with the returned idempotencyKey. Pending work is not started again; a saved response replays. Unknown work permits compensation only."}}),message:c==="pending"?"This operation is still running. Retry with the same idempotencyKey.":"The execution outcome is unknown. Retry with the same idempotencyKey for result or compensation recovery."}},a,c),s;try{s=await P({...e,idempotencyKey:t,leaseMs:3e5})}catch(a){return a instanceof d?n({status:400,body:{error_code:"invalid_request",message:a.message,retryable:!1}},null,"not_started"):a instanceof J?n({status:409,body:{error_code:"idempotency_conflict",message:a.message,retryable:!1}},null,"conflict"):r(null,"unknown")}let o=s.kind==="claimed"?s.claim.runId:s.runId;if(s.kind==="replay"){let a=JSON.parse(s.result.json);return!a||typeof a.status!="number"||!a.body||typeof a.body!="object"?r(o,"unknown"):n(a,o,"replayed")}if(s.kind==="pending")return r(o,"pending");if(s.kind==="unknown")try{let a=await y(o,e.userId);if(!a)return r(o,"unknown");let c=await e.recover(a);if(c.recover)return await h(a),n(c.reply,o,"unknown");let p=n(c.reply,o,"recovered_unknown"),l={json:JSON.stringify(p),status:"unknown"};return await O(a,l)?p:r(o,"unknown")}catch{return r(o,"unknown","refund_pending")}if(s.kind!=="claimed")return r(o,"unknown");let u=s.claim,f;try{if(f=await e.prepare(),f.rejection)return await I(u),n(f.rejection,o,"not_started");if(!await k(u))return r(o,"unknown");let a={claim:u,deadlineAt:i,assertActive:async()=>{if(B(i),!await m(u))throw Error("Execution ownership is no longer current")}},c;try{c=await e.execute(a,f.value)}catch{return await h(u),r(o,"unknown","refund_pending")}if(c.recover)return await h(u),n(c.reply,o,"unknown");let p=n(c.reply,o,c.reply.status<400?"succeeded":"failed");try{return await x(u,{json:JSON.stringify(p),status:p.status<400?"succeeded":"failed"})?p:r(o,"unknown")}catch{return await h(u).catch(()=>{}),r(o,"unknown")}}catch{return await I(u).catch(()=>{}),r(o,"unknown")}finally{await f?.release().catch(()=>{})}}var g=3e5;function ee(e){if(!e?.trim()||e.length>512)throw new d("A bounded framework workflow ID is required");return T(`workflow:${e}`)}async function ye(e){if(!Number.isSafeInteger(e.userId)||e.userId<=0)throw new d("Workflow principal is required");let i=ee(e.workflowId),t=await E(e,i,g);if(t.kind==="claimed"){let n;try{n=await k(t.claim)}catch(r){if(!await m(t.claim))throw r;n=!0}if(!n)throw Error("workflow_execution_start_unknown");return t}if(t.kind==="pending"){let n={runId:t.runId,userId:e.userId,token:i};if(await m(n)&&await S(n,g))return{kind:"claimed",claim:n}}return t}var te=e=>`${e.workflow.runId}:${e.phase}`;async function he(e){if(!await S(e.workflow,g))throw Error("workflow_execution_not_current");let i=te(e),t=await P({userId:e.workflow.userId,tool:e.tool,jobId:e.jobId,normalizedInput:e.normalizedInput,idempotencyKey:i,leaseMs:g});if(t.kind==="replay"){if(t.result.status!=="succeeded")throw Error(`operation_phase_unknown:${t.runId}`);return JSON.parse(t.result.json)}if(t.kind!=="claimed")throw Error(`operation_phase_${t.kind}:${t.runId}`);let n=t.claim;try{await e.prepare()}catch(r){throw await I(n).catch(()=>{}),r}try{let r;try{r=await k(n)}catch(a){if(!await m(n))throw a;r=!0}if(!r)throw Error("operation_phase_start_unknown");let s=Date.now()+g,o=async()=>{if(Date.now()>=s)throw new L(e.phase);if(!await m(e.workflow)||!await m(n))throw Error("operation_phase_claim_not_current")};await o();let u=await H(s,()=>e.execute({claim:n,deadlineAt:s,assertActive:o}),e.phase);await o();let f=JSON.stringify(u);if(f==null||!await x(n,{status:"succeeded",json:f}))throw Error("operation_phase_publication_unknown");return u}catch(r){await h(n).catch(()=>{});let s=await y(n.runId,n.userId).catch(()=>null);throw s&&await O(s,{status:"unknown",json:JSON.stringify({phase:e.phase,outcome:"unknown"})}).catch(()=>{}),r}}async function be(e,i){let t=await Y({userId:e.workflow.userId,tool:e.tool,idempotencyKey:te(e),normalizedInput:e.normalizedInput});if(!t||t.result.status!=="succeeded"||R({value:JSON.parse(t.result.json)})!==R({value:i}))throw Error("operation_phase_cache_receipt_unknown")}async function ge(e,i,t){let n={runId:e,userId:i,token:ee(t)};if(await m(n))return{claim:n,recovery:!1};let r=await y(e,i);return r?{claim:r,recovery:!0}:null}async function re(e,i,t){let n={json:JSON.stringify(i),status:e.recovery?"unknown":t};if(!(e.recovery?await O(e.claim,n,e.settlementGate):await x(e.claim,n)))throw Error("workflow_result_publication_unknown")}async function je(e){if(!await b(e))return{kind:"missing",body:{error:"job not found"}};let t=await M(e);if(!t)return{kind:"conflict",body:{error:"Only failed or expired recovery jobs can be re-finalized.",status:(await b(e))?.status??"missing"}};let n=t.executionGeneration;if(!await D(e,n))return{kind:"conflict",body:{error:"job state changed during re-finalization claim"}};let r;try{r=await X(t,{})}catch(l){throw await F(e,l instanceof Error?l.message:String(l),void 0,n).catch(()=>!1),l}let s=A(t),o=$(t,s.creditTruncated),u=o==="failed"?`No pages were extracted successfully (${t.failedUrls} failed, ${t.remainingUrls} remaining).`:o==="partial"?`${t.successfulUrls} pages succeeded; ${t.failedUrls} failed and ${t.remainingUrls} remain.${s.creditTruncated?` Credit availability limited this request from ${s.requestedMaxPages} to ${s.effectiveMaxPages} pages.`:""}`:null,f;try{f=await q(e,r,o,u,!1,null,n)}catch(l){let w=await b(e).catch(()=>null);if(!w||w.id!==e||w.executionGeneration!==n||w.status!==o||JSON.stringify(w.artifacts)!==JSON.stringify(r)||w.error!==(u?z(u).slice(0,2e3):null)||w.publicError!==null)throw l;f=!0}if(!f)return{kind:"conflict",body:{error:"job state changed during re-finalization"}};let a=await b(e);if(!a||a.executionGeneration!==n||a.status!==o)return{kind:"conflict",body:{error:"job state changed before re-finalization settlement"}};let c="already_settled_or_refunded";if(a.billedMc==null&&a.userId!=null){let l=await j(a);if(!l)throw Error("extract_settlement_acknowledgment_unknown");c=`settled: billed ${l.netChargeMc} mc, refunded ${l.refundMc} mc${l.expiredRefundMc?`, expired refund ${l.expiredRefundMc} mc`:""}`}let p=r.find(l=>l.contentType==="application/zip"||l.filename?.endsWith(".zip"));return{kind:"complete",body:{ok:!0,jobId:e,artifacts:r.length,settlement:c,bundleUrl:p?.url??null,bundleBytes:p?.bytes??null}}}async function Pe(e=25){let i=await U(e);for(let r of i)await G(r.id).catch(()=>!1);let t=await C(e),n={checked:new Set([...i,...t].map(r=>r.id)).size,settled:0,failed:0};for(let r of t){if(r.userId==null){await W(r.id,"Settlement abandoned because the legacy job has no user owner.",r.executionGeneration??0),n.failed+=1;continue}try{if(!await j(r))throw Error("extract_settlement_acknowledgment_unknown");if(r.executionGeneration===0&&await N(`extract-site:${r.id}`,r.userId)){let o=await y(`extract-site:${r.id}`,r.userId);o&&await re({claim:o,recovery:!0},{jobId:r.id,status:r.status,artifacts:r.artifacts??[]},"unknown")}n.settled+=1}catch(s){n.failed+=1;let o=s instanceof Error?s.message:String(s);await K(r.id,o,r.executionGeneration??0).catch(()=>{}),console.warn("[site-extract/reconcile]",r.id,o)}}return n}export{we as a,g as b,ee as c,ye as d,he as e,be as f,ge as g,re as h,je as i,Pe as j};

package/dist/chunk-UXNR3K6I.js

@@ -1 +0,0 @@
-import{g as c,j as d,k as p,l as w,m}from"./chunk-7WPCGRD7.js";import{i as u}from"./chunk-C7MGJUTU.js";import{Inngest as x}from"inngest";var h=new x({id:"mcp-scraper"});function b(r){return Math.min(3600,Math.max(15,15*2**Math.min(8,Math.max(0,r-1))))}async function f(r){let t={claimed:r.rows.length,dispatched:0,failed:0},e=()=>{t.unknown=(t.unknown??0)+1},o=async a=>{let i=await r.read(a).catch(()=>null);return i?.id===a.id?i:null};for(let a of r.rows){let i=a.claimToken;if(!i||!await r.isCurrent(a,i).catch(()=>!1)){e();continue}try{await r.send(a)}catch(n){let l=(n instanceof Error?n.message:String(n)).slice(0,2e3);if(await r.retainRetry(a,i,l,b(a.attempts)).catch(async()=>{let s=await o(a);return s?.status==="failed"&&s.lastError===l}))t.failed+=1;else{let s=await o(a);s?.status==="dispatched"&&s.dispatchedAt?t.dispatched+=1:e()}continue}if(await r.acknowledge(a,i).catch(()=>!1))t.dispatched+=1;else{let n=await o(a);n?.status==="dispatched"&&n.dispatchedAt?t.dispatched+=1:e()}}return t}async function k(r){return f({rows:r,isCurrent:(t,e)=>p(t.id,e),send:async t=>{try{return await h.send({id:t.id,name:"mcp-scraper/extract.requested",data:{jobId:t.id}})}catch(e){throw Error(u(e instanceof Error?e.message:String(e)).slice(0,2e3))}},acknowledge:(t,e)=>w(t.id,e),retainRetry:(t,e,o,a)=>m(t.id,e,o,a),read:async t=>{let e=await c(t.id);return e?{id:e.id,status:e.dispatchedAt?"dispatched":e.dispatchClaimToken?"dispatching":e.dispatchError?"failed":"pending",dispatchedAt:e.dispatchedAt??null,lastError:e.dispatchError??null}:null}})}async function A(r){if((await c(r))?.dispatchedAt)return{claimed:0,dispatched:1,failed:0};let e=await d({jobId:r});return e.length?k(e):{claimed:0,dispatched:0,failed:0,unknown:1}}async function S(r=25){let t=await k(await d({limit:r}));return{checked:t.claimed,dispatched:t.dispatched,failed:t.failed,...t.unknown?{unknown:t.unknown}:{}}}export{h as a,f as b,A as c,S as d};

package/dist/dispatch-P7IY75LH.js

@@ -1 +0,0 @@
-import{c as a,d as b}from"./chunk-UXNR3K6I.js";import"./chunk-7WPCGRD7.js";import"./chunk-KXJCGV3H.js";import"./chunk-HUNSEGUN.js";import"./chunk-SEZETSB3.js";import"./chunk-SDQW7IJN.js";import"./chunk-C7MGJUTU.js";import"./chunk-4ZVO6GX4.js";import"./chunk-WNUXMPVX.js";import"./chunk-OILOHQNL.js";import"./chunk-HNNYXMSI.js";export{a as dispatchSiteExtractJob,b as redispatchFundedSiteExtractJobs};

package/dist/metering-ZH2P62IC.js

@@ -1 +0,0 @@
-import{a,b,c,d,e,f,g}from"./chunk-5BKSKLHX.js";import"./chunk-7SR7MEU5.js";import"./chunk-HUNSEGUN.js";import"./chunk-4ZVO6GX4.js";import"./chunk-WNUXMPVX.js";import"./chunk-HNNYXMSI.js";export{g as finishExtractUrlCostRun,b as finishMeteredProviderAction,d as linkCurrentCostDebit,f as startExtractUrlCostRun,a as startMeteredProviderAction,c as withLocalCostPhase,e as withLogicalCostRun};

package/dist/queue-AC4XKD6Z.js

@@ -1 +0,0 @@
-import{t as d,u,v as l,w as f,x as v,xb as S,zb as h}from"./chunk-H6MODA7W.js";import"./chunk-SFUOSJ3A.js";import"./chunk-NNOX3FYG.js";import"./chunk-NALXKTUU.js";import"./chunk-6BD75VKS.js";import"./chunk-2IATUJO6.js";import"./chunk-CORWWDHL.js";import"./chunk-5LYINW7Z.js";import"./chunk-VVSHZV5S.js";import"./chunk-4ZVO6GX4.js";import"./chunk-WNUXMPVX.js";import"./chunk-WP3QS4CJ.js";var g=12;function b(e){return e&&typeof e=="object"&&"code"in e&&typeof e.code=="string"?e.code:e instanceof Error?e.name:"unknown_error"}function k(e){return Math.min(21600,30*2**Math.max(0,e-1))}async function x(e={}){let w=e.stripeClient??S,E=Math.max(1,Math.min(100,e.limit??100)),r=500,i=0,s=0,c=0,m=0;for(;i<r;){let a=await d(Math.min(E,r-i),e.leaseSeconds??300);if(a.length===0)break;i+=a.length;for(let t of a)try{let n=await w.events.retrieve(t.eventId);await h(n),await u(t.eventId,t.token)&&(s+=1)}catch(n){let p=n instanceof Error?n.message:String(n),o=b(n);t.attemptCount>=g||o==="resource_missing"?await f(t.eventId,t.token,p,o)&&(m+=1):await l(t.eventId,t.token,p,o,k(t.attemptCount))&&(c+=1)}}return{claimed:i,processed:s,retried:c,deadLettered:m,capped:i>=r,health:await v()}}export{x as drainStripeEventQueue};

package/dist/reconciliation-UZUZ6B74.js

@@ -1 +0,0 @@
-import{i as b,j as c}from"./chunk-RYX4FJVH.js";import{d as a}from"./chunk-UXNR3K6I.js";import"./chunk-BPKTUCV7.js";import"./chunk-6BD75VKS.js";import"./chunk-ZTMQEDIB.js";import"./chunk-U4WIJZCX.js";import"./chunk-7WPCGRD7.js";import"./chunk-KXJCGV3H.js";import"./chunk-JMZIDHYQ.js";import"./chunk-M5QHXNFZ.js";import"./chunk-QER2ELZP.js";import"./chunk-CORWWDHL.js";import"./chunk-5LYINW7Z.js";import"./chunk-CNSVZTTT.js";import"./chunk-5BKSKLHX.js";import"./chunk-7SR7MEU5.js";import"./chunk-HUNSEGUN.js";import"./chunk-SEZETSB3.js";import"./chunk-SDQW7IJN.js";import"./chunk-C7MGJUTU.js";import"./chunk-4ZVO6GX4.js";import"./chunk-WNUXMPVX.js";import"./chunk-WP3QS4CJ.js";import"./chunk-OLYPLX5C.js";import"./chunk-B3JRL45T.js";import"./chunk-OILOHQNL.js";import"./chunk-HNNYXMSI.js";export{c as reconcileSiteExtractSettlements,a as redispatchFundedSiteExtractJobs,b as refinalizeExtractJob};

package/dist/repository-4EC5EXVN.js

@@ -1 +0,0 @@
-import{a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t}from"./chunk-HUNSEGUN.js";import"./chunk-4ZVO6GX4.js";import"./chunk-WNUXMPVX.js";import"./chunk-HNNYXMSI.js";export{a as claimExecution,d as claimOperationRecovery,k as createAdmittedOperation,l as finishOperationRun,o as hasOperationExecution,p as hasUnknownOperationGroup,i as isExecutionClaimCurrent,h as markExecutionUnknown,q as noActiveOperationGroupFence,t as noUnresolvedOperationGroupFence,m as operationExecutionFence,f as publishOperationRecovery,e as publishOperationResult,n as readOperationIntentResult,s as readStoredOperationIntent,g as releaseExecutionClaim,c as renewExecutionClaim,b as startExecutionEffects,r as startGuardedExecutionEffects,j as startOperationRun};

package/dist/settlement-6CS4JZTY.js

@@ -1 +0,0 @@
-import{b as a,c as b}from"./chunk-BPKTUCV7.js";import"./chunk-6BD75VKS.js";import"./chunk-7WPCGRD7.js";import"./chunk-KXJCGV3H.js";import"./chunk-CORWWDHL.js";import"./chunk-5LYINW7Z.js";import"./chunk-HUNSEGUN.js";import"./chunk-SEZETSB3.js";import"./chunk-SDQW7IJN.js";import"./chunk-C7MGJUTU.js";import"./chunk-4ZVO6GX4.js";import"./chunk-WNUXMPVX.js";import"./chunk-WP3QS4CJ.js";import"./chunk-OILOHQNL.js";import"./chunk-HNNYXMSI.js";export{a as settleExtractJobCharge,b as settleTerminalExtractJob};

package/dist/worker-HHUU2ECG.js

@@ -1 +0,0 @@
-import{N as a,O as b,P as c}from"./chunk-2OKQVTQ5.js";import"./chunk-6BD75VKS.js";import"./chunk-U4WIJZCX.js";import"./chunk-HVQZRQRJ.js";import"./chunk-M5QHXNFZ.js";import"./chunk-JHT6A4U6.js";import"./chunk-63VWBMYR.js";import"./chunk-5LYINW7Z.js";import"./chunk-7SR7MEU5.js";import"./chunk-HUNSEGUN.js";import"./chunk-B4ZKPLSP.js";import"./chunk-VVSHZV5S.js";import"./chunk-C7MGJUTU.js";import"./chunk-4ZVO6GX4.js";import"./chunk-WNUXMPVX.js";import"./chunk-6N5EC7ZO.js";import"./chunk-B3JRL45T.js";import"./chunk-HNNYXMSI.js";export{b as drainQueue,c as startWorker,a as tickOnce};