mcp-scraper 0.97.0 → 0.98.0

package/dist/chunk-4ZVO6GX4.js

@@ -1 +0,0 @@
-import{AsyncLocalStorage as n}from"async_hooks";var s="MCP_SCRAPER_ALLOW_HEADLESS_HARVEST_TEST",t=new n;function u(e,o){return t.run(e,o)}function l(){return t.getStore()}export{s as a,u as b,l as c};

package/dist/chunk-55ZANTHL.js

@@ -1,3 +0,0 @@
-import{e as _,j as y}from"./chunk-SEZETSB3.js";import{i as w}from"./chunk-SDQW7IJN.js";import{b as A}from"./chunk-B3JRL45T.js";import{c as h}from"./chunk-OILOHQNL.js";import{join as S}from"path";import{randomBytes as B}from"crypto";import{ZipFile as D}from"yazl";import v from"p-limit";var E="maps-media/",N=10080*60*1e3,L=900*1e3,m=5*1024*1024,O=40*1024*1024,U=15e5,k=25e5,j=6,I=4;function T(){return process.env.MAPS_MEDIA_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim()||null}function b(){return{prefix:E,artifactTtlMs:N,downloadTtlMs:L,token:T()}}function f(e){try{let t=new URL(e);if(t.protocol!=="https:")return null;let r=t.hostname.toLowerCase().replace(/\.$/,"");return r!=="googleusercontent.com"&&!r.endsWith(".googleusercontent.com")&&r!=="streetviewpixels-pa.googleapis.com"?null:t}catch{return null}}function F(e){let t=f(e);return t?(t.hostname.toLowerCase().endsWith("googleusercontent.com")&&/=[^/]+$/.test(t.pathname)&&(t.pathname=t.pathname.replace(/=[^/]+$/,"=s2048")),t.href):e}function $(e){return e.length>=3&&e[0]===255&&e[1]===216&&e[2]===255?{mimeType:"image/jpeg",extension:"jpg"}:e.length>=8&&e[0]===137&&e[1]===80&&e[2]===78&&e[3]===71?{mimeType:"image/png",extension:"png"}:e.length>=12&&String.fromCharCode(...e.slice(0,4))==="RIFF"&&String.fromCharCode(...e.slice(8,12))==="WEBP"?{mimeType:"image/webp",extension:"webp"}:e.length>=6&&["GIF87a","GIF89a"].includes(String.fromCharCode(...e.slice(0,6)))?{mimeType:"image/gif",extension:"gif"}:null}async function G(e){let t=Number(e.headers.get("content-length")??"");if(Number.isFinite(t)&&t>m)throw new Error("image_too_large");if(!e.body){let s=new Uint8Array(await e.arrayBuffer());if(s.byteLength>m)throw new Error("image_too_large");return s}let r=e.body.getReader(),l=[],a=0;try{for(;;){let{done:s,value:c}=await r.read();if(s)break;if(c){if(a+=c.byteLength,a>m)throw new Error("image_too_large");l.push(c)}}}finally{r.releaseLock()}let o=new Uint8Array(a),d=0;for(let s of l)o.set(s,d),d+=s.byteLength;return o}async function W(e,t){let r=f(F(e));if(!r)throw new Error("image_host_not_allowed");for(let l=0;l<=I;l+=1){let a=await A(r.href,{field:"Maps image URL"});if(a.error||!a.parsed)throw new Error("image_url_rejected");let o=await fetch(a.parsed.href,{redirect:"manual",headers:{Accept:"image/jpeg,image/png,image/webp,image/gif;q=0.8"},signal:t?AbortSignal.any([t,AbortSignal.timeout(15e3)]):AbortSignal.timeout(15e3)});if([301,302,303,307,308].includes(o.status)){let c=o.headers.get("location");if(!c||l===I)throw new Error("image_redirect_rejected");if(r=f(new URL(c,r).href),!r)throw new Error("image_redirect_host_not_allowed");continue}if(!o.ok)throw new Error(`image_fetch_${o.status}`);let d=await G(o),s=$(d);if(!s)throw new Error("image_content_invalid");return{bytes:Buffer.from(d),...s,finalUrl:r.href}}throw new Error("image_redirect_rejected")}function z(e){return new Promise((t,r)=>{let l=new D,a=[];l.outputStream.on("data",o=>a.push(Buffer.from(o))),l.outputStream.once("error",r),l.outputStream.once("end",()=>t(Buffer.concat(a)));for(let o of e)l.addBuffer(o.content,o.path);l.end()})}function X(e){return e.toLowerCase().replace(/[^a-z0-9]+/g,"-").replace(/^-+|-+$/g,"").slice(0,70)||"google-maps-place"}function K(e){return w(e,E)}async function ee(e){return K(e.artifactId)!==e.ownerId?null:y({policy:b(),artifactId:e.artifactId,maxBytes:50*1024*1024})}async function te(e){let t=new AbortController,r=e.deadlineMs?setTimeout(()=>t.abort(new Error("maps_media_packaging_deadline")),Math.max(1,e.deadlineMs-Date.now())):null,l=v(j),a=[],o=0,d=0,s=0,c=e.media.images.map(n=>l(async()=>{if(!t.signal.aborted){try{let i=await W(n.sourceUrl,t.signal);if(o+i.bytes.length>O)throw new Error("archive_byte_limit_reached");o+=i.bytes.length;let g=`images/${String(n.index).padStart(4,"0")}-${n.origin}.${i.extension}`;a.push({path:g,content:i.bytes}),n.sourceUrl=i.finalUrl,n.filename=g,n.mimeType=i.mimeType,n.bytes=i.bytes.length,n.downloadStatus="downloaded",n.downloadError=null,s<e.maxInlineImages&&i.bytes.length<=U&&d+i.bytes.length<=k&&(n.inlinePreview={data:i.bytes.toString("base64"),mimeType:i.mimeType},d+=i.bytes.length,s+=1)}catch(i){if(t.signal.aborted)return;n.filename=null,n.mimeType=null,n.bytes=null,n.downloadStatus="failed",n.downloadError=i instanceof Error?i.message:"image_download_failed",n.inlinePreview=null}t.signal.aborted||await e.onImage?.(n)}}));try{if(await Promise.all(c),t.signal.aborted)throw new Error("maps_media_packaging_deadline")}catch(n){throw t.abort(n),await Promise.allSettled(c),n}finally{r&&clearTimeout(r)}e.media.imagesDownloaded=e.media.images.filter(n=>n.downloadStatus==="downloaded").length;let u=e.media.images.length-e.media.imagesDownloaded;if(u>0&&e.media.warnings.push(`${u} image(s) could not be downloaded; their manifest rows include a bounded error code.`),e.media.imagesDownloaded===0)return{media:e.media,artifact:null};let M=e.media.images.map(({inlinePreview:n,...i})=>i),x={businessName:e.businessName,placeUrl:e.placeUrl,extractedAt:e.extractedAt,scope:e.media.scope,imagesCollected:e.media.imagesCollected,imagesDownloaded:e.media.imagesDownloaded,ownerImagesCollected:e.media.ownerImagesCollected,otherImagesCollected:e.media.otherImagesCollected,unknownOriginImagesCollected:e.media.unknownOriginImagesCollected,ownerGalleryExhausted:e.media.ownerGalleryExhausted,galleryExhausted:e.media.exhausted,stopReason:e.media.stopReason,notes:["origin=owner is backed by the Google Maps By owner gallery or an exact media-key match to it.","origin=other means absent from an exhausted owner gallery; it can include customer, review, Street View, Google, or unattributed media.","Google-hosted photo files use a bounded s2048 rendition when that CDN form is available; each file is still subject to byte and signature validation."]};a.push({path:"summary.json",content:Buffer.from(JSON.stringify(x,null,2))}),a.push({path:"images.jsonl",content:Buffer.from(M.map(n=>JSON.stringify(n)).join(`
-`)+`
-`)});let R=await z(a),C=B(6).toString("hex"),p=await _({policy:b(),ownerId:e.ownerId,artifactKey:`${C}.zip`,createdAt:new Date(e.extractedAt),filename:`${X(e.businessName)}-maps-images.zip`,contentType:"application/zip",content:R}),P=T()?null:S(h(),"blobs",p.artifactId);return{media:e.media,artifact:{...p,localPath:P}}}export{E as a,N as b,b as c,K as d,ee as e,te as f};

package/dist/chunk-63VWBMYR.js

@@ -1,3 +0,0 @@
-import{e as l}from"./chunk-C7MGJUTU.js";import{a as o}from"./chunk-WNUXMPVX.js";function c(t){return{id:String(t.id),user_id:Number(t.user_id),status:String(t.status),query:String(t.query),options:String(t.options),callback_url:t.callback_url!=null?String(t.callback_url):null,result:t.result!=null?String(t.result):null,error:t.error!=null?String(t.error):null,public_error_json:t.public_error_json!=null?String(t.public_error_json):null,created_at:String(t.created_at),started_at:t.started_at!=null?String(t.started_at):null,completed_at:t.completed_at!=null?String(t.completed_at):null}}function g(t){let{public_error_json:r,...n}=t;return{...n,options:JSON.parse(t.options),result:t.result?JSON.parse(t.result):null,publicError:l(r)}}async function a(t,r){let n=o(),e=r?await n.execute({sql:"SELECT * FROM jobs WHERE id = ? AND user_id = ?",args:[t,r]}):await n.execute({sql:"SELECT * FROM jobs WHERE id = ?",args:[t]});return e.rows[0]?g(c(e.rows[0])):void 0}async function b(t,r,n,e,s){if(!/^[a-zA-Z0-9-]{1,100}$/.test(t))throw new Error("job id is invalid");await o().execute({sql:"INSERT INTO jobs (id, user_id, query, options, callback_url) VALUES (?, ?, ?, ?, ?)",args:[t,r,n,JSON.stringify(e),s??null]})}async function _(t,r,n,e,s){let u=Number(r);if(!Number.isSafeInteger(u)||u<=0||typeof e.requestFingerprint!="string"||!/^[a-f0-9]{64}$/.test(e.requestFingerprint))throw new Error("Job receipt requires a principal and request fingerprint");try{return await b(t,r,n,e,s),{kind:"created",jobId:t}}catch{let i;try{i=await a(t,r)}catch{return{kind:"unknown",jobId:t}}return!i||i.query!==n||i.callback_url!==(s??null)||i.options.requestFingerprint!==e.requestFingerprint||i.options.billingDebitKey!==e.billingDebitKey||i.options.billingHoldMc!==e.billingHoldMc?{kind:"unknown",jobId:t}:{kind:"recovered",jobId:t,job:i}}}async function f(t,r,n){let e=Number(r);if(!Number.isSafeInteger(e)||e<=0||!/^[a-f0-9]{64}$/.test(n))throw new Error("Job dispatch receipt requires a principal and request fingerprint");try{if((await o().execute({sql:`UPDATE jobs SET options=json_set(options,'$.dispatchRequestedAt',strftime('%Y-%m-%dT%H:%M:%fZ','now'))
-        WHERE id=? AND user_id=? AND json_extract(options,'$.requestFingerprint')=?
-          AND json_extract(options,'$.dispatchRequestedAt') IS NULL`,args:[t,r,n]})).rowsAffected===1)return!0}catch{}let s=await a(t,r).catch(()=>{});return s?.options.requestFingerprint===n&&typeof s.options.dispatchRequestedAt=="string"}export{c as a,g as b,a as c,b as d,_ as e,f};

package/dist/chunk-6FAMBNHY.js

@@ -1 +0,0 @@
-var o="0.97.0";export{o as a};

package/dist/chunk-6N5EC7ZO.js

@@ -1 +0,0 @@
-async function t(o){if((process.env.NODE_ENV==="test"||process.env.VITEST)&&process.env.MCP_SCRAPER_ENABLE_MEMORY_LIBRARY_SINK_IN_TESTS!=="1")return;let r=process.env.MCP_SCRAPER_API_KEY?.trim(),n=(process.env.MCP_SCRAPER_BASE_URL??"https://mcpscraper.dev").replace(/\/$/,"");if(r)try{let e=await fetch(`${n}/memory/mcp-call`,{method:"POST",headers:{"content-type":"application/json","x-api-key":r},body:JSON.stringify({toolName:"libraryIngestTool",args:o})}),s=await e.json().catch(()=>null);(!e.ok||s?.ok===!1)&&console.warn("[memory-library-sink] ingest not accepted:",e.status,s?.error??"")}catch(e){console.warn("[memory-library-sink] ingest failed:",e?.message)}}export{t as a};

package/dist/chunk-7POHNTVW.js

@@ -1 +0,0 @@
-import{AsyncLocalStorage as n}from"async_hooks";var r=new n;function o(t,e){return r.run(t,e)}function i(){return r.getStore()}export{o as a,i as b};

package/dist/chunk-7SR7MEU5.js

@@ -1,62 +0,0 @@
-import{a as o}from"./chunk-WNUXMPVX.js";import{c,d as p,e as a}from"./chunk-HNNYXMSI.js";import{randomUUID as m}from"crypto";async function O(e){let r=m();return await o().execute({sql:`INSERT INTO operation_coverage_defects (id, tool, method, phase, error_code, run_id, attempt_id)
-      VALUES (?, ?, ?, ?, ?, ?, ?)`,args:[r,a(e.tool,"tool"),a(e.method,"method"),a(e.phase,"phase"),a(e.errorCode,"errorCode").slice(0,200),e.runId??null,e.attemptId??null]}),r}function l(e,r=2e3){return e==null?null:e.slice(0,r)}async function I(e){if(!Number.isInteger(e.attemptNumber)||e.attemptNumber<1)throw new Error("attemptNumber must be a positive integer");if(e.attemptNumber===1&&(e.parentAttemptId||e.edgeKind||e.retryKind))throw new Error("The first attempt cannot have a parent or retry edge");if(e.attemptNumber>1&&(!e.parentAttemptId||!e.edgeKind))throw new Error("Every non-first attempt requires a parentAttemptId and edgeKind");if(e.retryKind&&!["same_method_retry","provider_fallback","durable_resume","reconciliation"].includes(e.edgeKind??""))throw new Error("retryKind requires a retry or reconciliation edgeKind");let r=e.id??m();return await o().execute({sql:`INSERT INTO operation_attempts
-      (id, run_id, parent_attempt_id, attempt_number, edge_kind, retry_kind, retry_reason, retry_owner,
-       method, provider, normalized_flags_json, started_at)
-      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,args:[r,e.runId,e.parentAttemptId??null,e.attemptNumber,e.edgeKind??null,e.retryKind??null,l(e.retryReason,500),l(e.retryOwner,100),a(e.method,"method"),e.provider??null,p(e.normalizedFlags),e.startedAt??new Date().toISOString()]}),r}async function C(e){let r=m();return await o().execute({sql:`INSERT INTO operation_attempts
-      (id, run_id, parent_attempt_id, attempt_number, edge_kind, retry_kind, retry_reason,
-       method, provider, normalized_flags_json, started_at)
-      SELECT ?, ?, ?, COALESCE(MAX(attempt_number), 0) + 1, ?, ?, ?, ?, ?, ?, ?
-      FROM operation_attempts WHERE run_id = ?`,args:[r,e.runId,e.parentAttemptId,e.edgeKind,e.retryKind??null,l(e.retryReason,500),a(e.method,"method"),e.provider??null,p(e.normalizedFlags),new Date().toISOString(),e.runId]}),r}async function T(e){if(!Number.isInteger(e.durationMs)||e.durationMs<0)throw new Error("durationMs must be a non-negative integer");let r=e.endedAt??new Date().toISOString();if((await o().execute({sql:`UPDATE operation_attempts SET
-      status = ?, error_code = ?, error_message = ?, ended_at = ?, duration_ms = ?, updated_at = ?
-      WHERE id = ? AND status = 'running'`,args:[e.status,l(e.errorCode,200),l(e.errorMessage),r,e.durationMs,r,e.attemptId]})).rowsAffected===1)return;let i=await o().execute({sql:"SELECT status, duration_ms FROM operation_attempts WHERE id = ?",args:[e.attemptId]});if(i.rows.length===0)throw new Error(`Unknown operation attempt: ${e.attemptId}`);if(String(i.rows[0]?.status)!==e.status||Number(i.rows[0]?.duration_ms)!==e.durationMs)throw new Error(`Operation attempt ${e.attemptId} is already terminal as ${String(i.rows[0]?.status)}`)}async function L(e){let r=a(e.provider,"provider"),t=a(e.method,"method"),i=c(a(e.accountScope,"accountScope")),s=e.providerSourceKey?.trim()||null,n=s?c(s):null,d=e.id??m(),u=await o().execute({sql:`INSERT OR IGNORE INTO provider_usage_receipts
-      (id, attempt_id, provider, account_scope_hash, method, provider_product, billing_tier,
-       provider_source_key, provider_source_key_hash, provider_status)
-      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,args:[d,e.attemptId,r,i,t,e.providerProduct??null,e.billingTier??null,s,n,e.providerStatus??null]});if(!n){if(u.rowsAffected===1)return{id:d,replayed:!1};let _=(await o().execute({sql:"SELECT attempt_id, provider, method FROM provider_usage_receipts WHERE id = ?",args:[d]})).rows[0];if(!_||String(_.attempt_id)!==e.attemptId||String(_.provider)!==r||String(_.method)!==t)throw new Error("Provider receipt ID is already used by a different receipt");return{id:d,replayed:!0}}let g=(await o().execute({sql:`SELECT id, attempt_id FROM provider_usage_receipts
-      WHERE provider = ? AND account_scope_hash = ? AND method = ? AND provider_source_key_hash = ?`,args:[r,i,t,n]})).rows[0];if(!g)throw new Error("Provider receipt insert did not produce a source receipt");if(String(g.attempt_id)!==e.attemptId)throw new Error("Provider source identity is already attached to a different attempt");return{id:String(g.id),replayed:String(g.id)!==d}}async function x(e,r){let t=a(r,"providerSourceKey"),i=c(t);if((await o().execute({sql:`UPDATE provider_usage_receipts
-      SET provider_source_key = ?, provider_source_key_hash = ?, updated_at = datetime('now')
-      WHERE id = ? AND provider_source_key IS NULL`,args:[t,i,e]})).rowsAffected===1)return;let n=await o().execute({sql:"SELECT provider_source_key_hash FROM provider_usage_receipts WHERE id = ?",args:[e]});if(!n.rows[0])throw new Error(`Unknown provider receipt: ${e}`);if(String(n.rows[0].provider_source_key_hash)!==i)throw new Error(`Provider receipt ${e} already has a different source identity`)}async function v(e){if(!Number.isSafeInteger(e.units)||e.units<0)throw new Error("Observed units must be a non-negative safe integer");if((await o().execute({sql:`UPDATE provider_usage_receipts SET units = ?, unit_type = ?,
-      provider_status = COALESCE(?, provider_status), checked_at = ?, updated_at = datetime('now')
-      WHERE id = ? AND cost_state = 'pending'`,args:[e.units,a(e.unitType,"unitType"),e.providerStatus??null,e.checkedAt??new Date().toISOString(),e.receiptId]})).rowsAffected!==1)throw new Error(`Unknown or terminal pending provider receipt: ${e.receiptId}`)}async function k(e){let r=e.checkedAt??new Date().toISOString();if((await o().execute({sql:`UPDATE provider_usage_receipts SET checked_at = ?, updated_at = ?,
-      error_code = ?, provider_status = COALESCE(?, provider_status)
-      WHERE id = ? AND cost_state IN ('pending', 'estimated')`,args:[r,r,l(e.errorCode,200),l(e.providerStatus,200),e.receiptId]})).rowsAffected===1)return;if(!(await o().execute({sql:"SELECT id FROM provider_usage_receipts WHERE id = ?",args:[e.receiptId]})).rows[0])throw new Error(`Unknown provider receipt: ${e.receiptId}`)}async function P(e){let t=(await o().execute({sql:`SELECT id FROM provider_usage_receipts
-      WHERE provider = ? AND account_scope_hash = ? AND method = ? AND provider_source_key_hash = ?`,args:[a(e.provider,"provider"),c(a(e.accountScope,"accountScope")),a(e.method,"method"),c(a(e.providerSourceKey,"providerSourceKey"))]})).rows[0];return t?(await v({receiptId:String(t.id),units:e.units,unitType:e.unitType,providerStatus:e.providerStatus,checkedAt:e.checkedAt}),!0):!1}async function S(e){if(e.units!=null&&(!Number.isFinite(e.units)||e.units<0))throw new Error("units must be non-negative");if(e.amountUsdNanos!=null&&(!Number.isSafeInteger(e.amountUsdNanos)||e.amountUsdNanos<0))throw new Error("amountUsdNanos must be a non-negative safe integer");if((e.costState==="actual"||e.costState==="measured")&&e.amountUsdNanos==null)throw new Error(`${e.costState} receipts require amountUsdNanos`);if((e.costState==="actual"||e.costState==="measured")&&e.amountUsdNanos===0&&!e.zeroCostReason?.trim())throw new Error(`${e.costState} zero-cost receipts require zeroCostReason`);if(e.costState==="measured"&&e.amountUsdNanos!==0&&(!e.providerProduct||!e.billingTier||!e.billableEvent||!e.measurementSource||!e.rateSource||e.rateNumerator==null||e.rateDenominator==null||!e.ratePolicyVersion||!e.rateEffectiveAt||e.units==null||!e.unitType))throw new Error("measured receipts require observed product, units, and an applied rate snapshot");if(e.rateNumerator!=null&&(!Number.isSafeInteger(e.rateNumerator)||e.rateNumerator<0))throw new Error("rateNumerator must be a non-negative safe integer");if(e.rateDenominator!=null&&(!Number.isSafeInteger(e.rateDenominator)||e.rateDenominator<1))throw new Error("rateDenominator must be a positive safe integer");let r=e.checkedAt??new Date().toISOString(),t=e.finalizedAt??r;if((await o().execute({sql:`UPDATE provider_usage_receipts SET
-      cost_state = ?,
-      units = CASE WHEN ? = 'actual' THEN ? ELSE COALESCE(?, units) END,
-      unit_type = CASE WHEN ? = 'actual' THEN ? ELSE COALESCE(?, unit_type) END,
-      amount_usd_nanos = ?, rate_source = COALESCE(?, rate_source), zero_cost_reason = ?,
-      provider_product = COALESCE(provider_product, ?), billing_tier = COALESCE(billing_tier, ?),
-      billable_event = CASE WHEN ? = 'actual' THEN ? ELSE COALESCE(?, billable_event) END,
-      measurement_source = COALESCE(?, measurement_source),
-      rate_numerator = CASE WHEN ? = 'actual' THEN ? ELSE COALESCE(?, rate_numerator) END,
-      rate_denominator = CASE WHEN ? = 'actual' THEN ? ELSE COALESCE(?, rate_denominator) END,
-      rate_policy_version = CASE WHEN ? = 'actual' THEN ? ELSE COALESCE(?, rate_policy_version) END,
-      rate_effective_at = CASE WHEN ? = 'actual' THEN ? ELSE COALESCE(?, rate_effective_at) END,
-      provider_status = ?, provider_duration_ms = ?, provider_captcha = ?, error_code = ?, error_message = ?,
-      checked_at = ?, finalized_at = ?, updated_at = ?
-      WHERE id = ? AND (cost_state = 'pending'
-        OR (cost_state = 'estimated' AND ? IN ('measured', 'actual'))
-        OR (cost_state = 'measured' AND ? = 'actual'))`,args:[e.costState,e.costState,e.units??null,e.units??null,e.costState,e.unitType??null,e.unitType??null,e.amountUsdNanos??null,e.rateSource??null,e.zeroCostReason??null,e.providerProduct??null,e.billingTier??null,e.costState,e.billableEvent??null,e.billableEvent??null,e.measurementSource??null,e.costState,e.rateNumerator??null,e.rateNumerator??null,e.costState,e.rateDenominator??null,e.rateDenominator??null,e.costState,e.ratePolicyVersion??null,e.ratePolicyVersion??null,e.costState,e.rateEffectiveAt??null,e.rateEffectiveAt??null,e.providerStatus??null,e.providerDurationMs??null,e.providerCaptcha==null?null:e.providerCaptcha?1:0,l(e.errorCode,200),l(e.errorMessage),r,t,t,e.receiptId,e.costState,e.costState]})).rowsAffected===1)return;let s=await o().execute({sql:"SELECT cost_state, units, unit_type, amount_usd_nanos FROM provider_usage_receipts WHERE id = ?",args:[e.receiptId]});if(s.rows.length===0)throw new Error(`Unknown provider receipt: ${e.receiptId}`);let n=s.rows[0],d=n?.units==null&&e.units==null||Number(n?.units)===e.units,u=n?.amount_usd_nanos==null&&e.amountUsdNanos==null||Number(n?.amount_usd_nanos)===e.amountUsdNanos,E=n?.unit_type==null&&e.unitType==null||String(n?.unit_type)===e.unitType;if(String(n?.cost_state)!==e.costState||!d||!u||!E)throw new Error(`Provider receipt ${e.receiptId} is already finalized as ${String(s.rows[0]?.cost_state)}`)}async function U(e){let t=(await o().execute({sql:`SELECT id FROM provider_usage_receipts
-      WHERE provider = ? AND account_scope_hash = ? AND method = ? AND provider_source_key_hash = ?`,args:[a(e.provider,"provider"),c(a(e.accountScope,"accountScope")),a(e.method,"method"),c(a(e.providerSourceKey,"providerSourceKey"))]})).rows[0];if(!t)return!1;let{provider:i,accountScope:s,method:n,providerSourceKey:d,...u}=e;return await S({...u,receiptId:String(t.id)}),!0}async function D(e=30){if(!Number.isInteger(e)||e<1||e>365)throw new Error("retentionDays must be an integer between 1 and 365");let r=`-${e} days`,t=o(),[i,s,n]=await Promise.all([t.execute({sql:`UPDATE operation_attempts
-        SET error_message = NULL, updated_at = datetime('now')
-        WHERE ended_at < datetime('now', ?) AND error_message IS NOT NULL`,args:[r]}),t.execute({sql:`UPDATE provider_usage_receipts
-        SET provider_source_key = NULL, error_message = NULL, updated_at = datetime('now')
-        WHERE finalized_at < datetime('now', ?)
-          AND (provider_source_key IS NOT NULL OR error_message IS NOT NULL)`,args:[r]}),t.execute({sql:`UPDATE harvest_attempts
-        SET provider_session_id = NULL, \u006b\u0065\u0072\u006e\u0065\u006c_session_id = NULL, error = NULL,
-            \u006b\u0065\u0072\u006e\u0065\u006c_delete_error = NULL, browser_close_error = NULL,
-            provider_error_message = NULL, provider_detail_json = NULL,
-            provider_disconnect_message = NULL, debug_json = NULL
-        WHERE completed_at < datetime('now', ?)
-          AND (provider_session_id IS NOT NULL OR \u006b\u0065\u0072\u006e\u0065\u006c_session_id IS NOT NULL OR error IS NOT NULL
-            OR \u006b\u0065\u0072\u006e\u0065\u006c_delete_error IS NOT NULL OR browser_close_error IS NOT NULL
-            OR provider_error_message IS NOT NULL OR provider_detail_json IS NOT NULL
-            OR provider_disconnect_message IS NOT NULL OR debug_json IS NOT NULL)`,args:[r]})]);return{operationAttempts:i.rowsAffected,providerReceipts:s.rowsAffected,legacyHarvestAttempts:n.rowsAffected}}async function H(e){let t={billing_authorizations:{column:"id"},billing_events:{column:"id"},billing_debits:{column:"idempotency_key"},billing_credits:{column:"idempotency_key"},ledger:{column:"id"}}[e.billingTable],i=a(e.billingKey,"billingKey"),s=await o().execute({sql:`SELECT b.user_id AS billing_user_id, r.user_id AS run_user_id
-      FROM operation_runs r
-      JOIN ${e.billingTable} b ON CAST(b.${t.column} AS TEXT) = ?
-      WHERE r.id = ?`,args:[i,e.runId]});if(!s.rows[0])throw new Error("Billing link target or operation run does not exist");let n=s.rows[0].billing_user_id,d=s.rows[0].run_user_id;if(d!=null&&n!=null&&String(d)!==String(n))throw new Error("Billing link owner does not match operation run owner");await o().execute({sql:`INSERT OR IGNORE INTO operation_billing_links
-      (run_id, billing_table, billing_key, relation, amount_mc) VALUES (?, ?, ?, ?, ?)`,args:[e.runId,e.billingTable,i,e.relation,e.amountMc??null]})}async function K(e,r=new Date().toISOString()){let t=m();return await o().execute({sql:"INSERT INTO provider_reconciliation_runs (id, provider, started_at) VALUES (?, ?, ?)",args:[t,a(e,"provider"),r]}),t}async function q(e){if([e.checkedCount,e.completedCount,e.pendingCount,e.unavailableCount,e.failedCount].some(s=>!Number.isInteger(s)||s<0))throw new Error("Reconciliation counts must be non-negative integers");if(e.completedCount+e.pendingCount+e.unavailableCount+e.failedCount>e.checkedCount)throw new Error("Reconciliation outcome counts cannot exceed checkedCount");let t=e.completedAt??new Date().toISOString();if((await o().execute({sql:`UPDATE provider_reconciliation_runs SET
-      status = ?, checked_count = ?, completed_count = ?, pending_count = ?, unavailable_count = ?,
-      failed_count = ?, error_code = ?, completed_at = ?
-      WHERE id = ? AND status = 'running'`,args:[e.status,e.checkedCount,e.completedCount,e.pendingCount,e.unavailableCount,e.failedCount,l(e.errorCode,200),t,e.id]})).rowsAffected!==1)throw new Error(`Unknown or terminal provider reconciliation run: ${e.id}`)}async function M(e){let r=o(),[t,i,s,n]=await Promise.all([r.execute({sql:"SELECT * FROM operation_runs WHERE id = ?",args:[e]}),r.execute({sql:"SELECT * FROM operation_attempts WHERE run_id = ? ORDER BY attempt_number",args:[e]}),r.execute({sql:`SELECT r.* FROM provider_usage_receipts r
-        JOIN operation_attempts a ON a.id = r.attempt_id
-        WHERE a.run_id = ? ORDER BY a.attempt_number, r.rowid`,args:[e]}),r.execute({sql:"SELECT * FROM operation_billing_links WHERE run_id = ? ORDER BY created_at, billing_table, billing_key",args:[e]})]);return t.rows[0]?{run:t.rows[0],attempts:[...i.rows],providerReceipts:[...s.rows],billingLinks:[...n.rows]}:null}export{O as a,I as b,C as c,T as d,L as e,x as f,v as g,k as h,P as i,S as j,U as k,D as l,H as m,K as n,q as o,M as p};

package/dist/chunk-A4UASMEQ.js

@@ -1 +0,0 @@
-import{c as j}from"./chunk-PD7BI7TR.js";import{k as X}from"./chunk-NULRGHJ4.js";import{A as F,c as A,d as $,g as b,h as C,o as K,p as U,q as G,r as M,s as D,t as W,z as q}from"./chunk-JNPM3U3L.js";import{a as B,b as L,c as H}from"./chunk-QER2ELZP.js";import{b as k,c as S,d as y,e as x,f as O,g as I,i as h,j as m,l as E,p as N}from"./chunk-ANQ4JJVI.js";import{h as v,i as z}from"./chunk-C7MGJUTU.js";import{a as d,b as J,c as T,d as R}from"./chunk-HNNYXMSI.js";import{randomUUID as oe}from"crypto";import{createHash as Q,randomUUID as V}from"crypto";function _(e,i=new Set){if(e===null||typeof e=="string"||typeof e=="boolean"||typeof e=="number"&&Number.isFinite(e))return e;if(typeof e!="object"||!e)throw new d("Operation inputs must be finite JSON data");if(i.has(e))throw new d("Operation inputs cannot be cyclic");if(!Array.isArray(e)&&![Object.prototype,null].includes(Object.getPrototypeOf(e)))throw new d("Operation inputs must be plain JSON data");i.add(e);try{return Array.isArray(e)?e.map(t=>_(t,i)):Object.fromEntries(Object.keys(e).sort().map(t=>[t,_(e[t],i)]))}finally{i.delete(e)}}async function P(e){if(!Number.isSafeInteger(e.userId)||e.userId<=0)throw new d("Authenticated operation principal is required");if(e.jobId!=null&&(!e.jobId.trim()||e.jobId.length>512))throw new d("Operation job ID is invalid");let i=e.tool.trim();if(!i||i.length>100)throw new d("Operation tool is required and bounded");if(e.idempotencyKey!=null&&(!e.idempotencyKey.trim()||e.idempotencyKey.length>512))throw new d("Operation idempotency key is invalid");let t=e.leaseMs??3e5;if(!Number.isSafeInteger(t)||t<1e3||t>36e5)throw new d("Execution lease must be between one second and one hour");let n=JSON.stringify(_(e.normalizedInput));if(Buffer.byteLength(n)>64*1024)throw new d("Operation inputs exceed 64 KiB");let r=e.idempotencyKey?.trim()??V();return E({userId:e.userId,ownerScope:`user:${e.userId}`,tool:i,jobId:e.jobId??null,normalizedFlags:e.normalizedInput,idempotencyKey:r,requestFingerprint:Q("sha256").update(n).digest("hex")},V(),t,e.storeGate)}async function Y(e){let i=JSON.stringify(_(e.normalizedInput));if(!Number.isSafeInteger(e.userId)||e.userId<=0||Buffer.byteLength(i)>64*1024)throw new d("Invalid operation receipt principal/input");let{readOperationIntentResult:t}=await import("./repository-UKDA4IWP.js"),{operationIdentityHash:n}=await import("./contracts-GUXGFCVV.js");return t(e.userId,n(`user:${e.userId}\0${e.tool}\0${e.idempotencyKey}`),Q("sha256").update(i).digest("hex"))}async function we(e){let i=Date.now()+27e4,t=e.idempotencyKey?.trim()??oe();if(!t||t.length>512)return{status:400,body:{error_code:"invalid_request",message:"Idempotency-Key must contain 1 to 512 characters"}};let n=(a,c,p)=>({...a,body:{...a.body,operation:{runId:c,idempotencyKey:t,state:p}}}),r=(a,c,p)=>n({status:c==="pending"?202:503,headers:{"Retry-After":"2"},body:{...v({errorCode:"response_lost",retryable:!0,chargeStatus:p,details:{retry_guidance:"Retry the identical request with the returned idempotencyKey. Pending work is not started again; a saved response replays. Unknown work permits compensation only."}}),message:c==="pending"?"This operation is still running. Retry with the same idempotencyKey.":"The execution outcome is unknown. Retry with the same idempotencyKey for result or compensation recovery."}},a,c),s;try{s=await P({...e,idempotencyKey:t,leaseMs:3e5})}catch(a){return a instanceof d?n({status:400,body:{error_code:"invalid_request",message:a.message,retryable:!1}},null,"not_started"):a instanceof J?n({status:409,body:{error_code:"idempotency_conflict",message:a.message,retryable:!1}},null,"conflict"):r(null,"unknown")}let o=s.kind==="claimed"?s.claim.runId:s.runId;if(s.kind==="replay"){let a=JSON.parse(s.result.json);return!a||typeof a.status!="number"||!a.body||typeof a.body!="object"?r(o,"unknown"):n(a,o,"replayed")}if(s.kind==="pending")return r(o,"pending");if(s.kind==="unknown")try{let a=await y(o,e.userId);if(!a)return r(o,"unknown");let c=await e.recover(a);if(c.recover)return await h(a),n(c.reply,o,"unknown");let p=n(c.reply,o,"recovered_unknown"),l={json:JSON.stringify(p),status:"unknown"};return await O(a,l)?p:r(o,"unknown")}catch{return r(o,"unknown","refund_pending")}if(s.kind!=="claimed")return r(o,"unknown");let u=s.claim,f;try{if(f=await e.prepare(),f.rejection)return await I(u),n(f.rejection,o,"not_started");if(!await k(u))return r(o,"unknown");let a={claim:u,deadlineAt:i,assertActive:async()=>{if(B(i),!await m(u))throw Error("Execution ownership is no longer current")}},c;try{c=await e.execute(a,f.value)}catch{return await h(u),r(o,"unknown","refund_pending")}if(c.recover)return await h(u),n(c.reply,o,"unknown");let p=n(c.reply,o,c.reply.status<400?"succeeded":"failed");try{return await x(u,{json:JSON.stringify(p),status:p.status<400?"succeeded":"failed"})?p:r(o,"unknown")}catch{return await h(u).catch(()=>{}),r(o,"unknown")}}catch{return await I(u).catch(()=>{}),r(o,"unknown")}finally{await f?.release().catch(()=>{})}}var g=3e5;function ee(e){if(!e?.trim()||e.length>512)throw new d("A bounded framework workflow ID is required");return T(`workflow:${e}`)}async function ye(e){if(!Number.isSafeInteger(e.userId)||e.userId<=0)throw new d("Workflow principal is required");let i=ee(e.workflowId),t=await E(e,i,g);if(t.kind==="claimed"){let n;try{n=await k(t.claim)}catch(r){if(!await m(t.claim))throw r;n=!0}if(!n)throw Error("workflow_execution_start_unknown");return t}if(t.kind==="pending"){let n={runId:t.runId,userId:e.userId,token:i};if(await m(n)&&await S(n,g))return{kind:"claimed",claim:n}}return t}var te=e=>`${e.workflow.runId}:${e.phase}`;async function he(e){if(!await S(e.workflow,g))throw Error("workflow_execution_not_current");let i=te(e),t=await P({userId:e.workflow.userId,tool:e.tool,jobId:e.jobId,normalizedInput:e.normalizedInput,idempotencyKey:i,leaseMs:g});if(t.kind==="replay"){if(t.result.status!=="succeeded")throw Error(`operation_phase_unknown:${t.runId}`);return JSON.parse(t.result.json)}if(t.kind!=="claimed")throw Error(`operation_phase_${t.kind}:${t.runId}`);let n=t.claim;try{await e.prepare()}catch(r){throw await I(n).catch(()=>{}),r}try{let r;try{r=await k(n)}catch(a){if(!await m(n))throw a;r=!0}if(!r)throw Error("operation_phase_start_unknown");let s=Date.now()+g,o=async()=>{if(Date.now()>=s)throw new L(e.phase);if(!await m(e.workflow)||!await m(n))throw Error("operation_phase_claim_not_current")};await o();let u=await H(s,()=>e.execute({claim:n,deadlineAt:s,assertActive:o}),e.phase);await o();let f=JSON.stringify(u);if(f==null||!await x(n,{status:"succeeded",json:f}))throw Error("operation_phase_publication_unknown");return u}catch(r){await h(n).catch(()=>{});let s=await y(n.runId,n.userId).catch(()=>null);throw s&&await O(s,{status:"unknown",json:JSON.stringify({phase:e.phase,outcome:"unknown"})}).catch(()=>{}),r}}async function be(e,i){let t=await Y({userId:e.workflow.userId,tool:e.tool,idempotencyKey:te(e),normalizedInput:e.normalizedInput});if(!t||t.result.status!=="succeeded"||R({value:JSON.parse(t.result.json)})!==R({value:i}))throw Error("operation_phase_cache_receipt_unknown")}async function ge(e,i,t){let n={runId:e,userId:i,token:ee(t)};if(await m(n))return{claim:n,recovery:!1};let r=await y(e,i);return r?{claim:r,recovery:!0}:null}async function re(e,i,t){let n={json:JSON.stringify(i),status:e.recovery?"unknown":t};if(!(e.recovery?await O(e.claim,n,e.settlementGate):await x(e.claim,n)))throw Error("workflow_result_publication_unknown")}async function je(e){if(!await b(e))return{kind:"missing",body:{error:"job not found"}};let t=await M(e);if(!t)return{kind:"conflict",body:{error:"Only failed or expired recovery jobs can be re-finalized.",status:(await b(e))?.status??"missing"}};let n=t.executionGeneration;if(!await D(e,n))return{kind:"conflict",body:{error:"job state changed during re-finalization claim"}};let r;try{r=await X(t,{})}catch(l){throw await F(e,l instanceof Error?l.message:String(l),void 0,n).catch(()=>!1),l}let s=A(t),o=$(t,s.creditTruncated),u=o==="failed"?`No pages were extracted successfully (${t.failedUrls} failed, ${t.remainingUrls} remaining).`:o==="partial"?`${t.successfulUrls} pages succeeded; ${t.failedUrls} failed and ${t.remainingUrls} remain.${s.creditTruncated?` Credit availability limited this request from ${s.requestedMaxPages} to ${s.effectiveMaxPages} pages.`:""}`:null,f;try{f=await q(e,r,o,u,!1,null,n)}catch(l){let w=await b(e).catch(()=>null);if(!w||w.id!==e||w.executionGeneration!==n||w.status!==o||JSON.stringify(w.artifacts)!==JSON.stringify(r)||w.error!==(u?z(u).slice(0,2e3):null)||w.publicError!==null)throw l;f=!0}if(!f)return{kind:"conflict",body:{error:"job state changed during re-finalization"}};let a=await b(e);if(!a||a.executionGeneration!==n||a.status!==o)return{kind:"conflict",body:{error:"job state changed before re-finalization settlement"}};let c="already_settled_or_refunded";if(a.billedMc==null&&a.userId!=null){let l=await j(a);if(!l)throw Error("extract_settlement_acknowledgment_unknown");c=`settled: billed ${l.netChargeMc} mc, refunded ${l.refundMc} mc${l.expiredRefundMc?`, expired refund ${l.expiredRefundMc} mc`:""}`}let p=r.find(l=>l.contentType==="application/zip"||l.filename?.endsWith(".zip"));return{kind:"complete",body:{ok:!0,jobId:e,artifacts:r.length,settlement:c,bundleUrl:p?.url??null,bundleBytes:p?.bytes??null}}}async function Pe(e=25){let i=await U(e);for(let r of i)await G(r.id).catch(()=>!1);let t=await C(e),n={checked:new Set([...i,...t].map(r=>r.id)).size,settled:0,failed:0};for(let r of t){if(r.userId==null){await W(r.id,"Settlement abandoned because the legacy job has no user owner.",r.executionGeneration??0),n.failed+=1;continue}try{if(!await j(r))throw Error("extract_settlement_acknowledgment_unknown");if(r.executionGeneration===0&&await N(`extract-site:${r.id}`,r.userId)){let o=await y(`extract-site:${r.id}`,r.userId);o&&await re({claim:o,recovery:!0},{jobId:r.id,status:r.status,artifacts:r.artifacts??[]},"unknown")}n.settled+=1}catch(s){n.failed+=1;let o=s instanceof Error?s.message:String(s);await K(r.id,o,r.executionGeneration??0).catch(()=>{}),console.warn("[site-extract/reconcile]",r.id,o)}}return n}export{P as a,we as b,g as c,ee as d,ye as e,he as f,be as g,ge as h,re as i,je as j,Pe as k};