mcp-scraper 0.95.2 → 0.96.1

package/dist/index.d.ts

@@ -3,10 +3,110 @@ import { z } from 'zod';
 declare const SERP_FEATURE_COMPLETENESS_STATES: readonly ["observed_present", "observed_absent", "incomplete", "not_requested", "unknown", "unsupported"];
 type SerpFeatureCompletenessState = typeof SERP_FEATURE_COMPLETENESS_STATES[number];
 
+type SerpLinkType = 'plain' | 'google_url_redirect' | 'google_goto_redirect';
+type SerpLinkResolutionStatus = 'not_needed' | 'resolved' | 'unresolved' | 'rejected';
+interface SerpOutboundLink {
+    url: string;
+    rawUrl: string;
+    resolvedUrl: string | null;
+    linkType: SerpLinkType;
+    resolutionStatus: SerpLinkResolutionStatus;
+}
+interface VideoResult {
+    type: 'video' | 'short_video';
+    title: string;
+    channel: string;
+    platform: string;
+    duration: string;
+    url: string;
+}
+interface ForumResult {
+    title: string;
+    source: string;
+    url: string;
+}
+interface WhatPeopleSayingCard {
+    type: 'reddit' | 'facebook' | 'instagram' | 'tiktok' | 'youtube' | 'news' | 'unknown';
+    title: string;
+    url: string;
+    source: string;
+    platform: string;
+    popularComment: string | null;
+    engagement: string;
+    date: string;
+    duration: string | null;
+    authorNote: string | null;
+}
+interface AICitation {
+    text: string;
+    href: string;
+    rawUrl: string;
+    resolvedUrl: string | null;
+    linkType: SerpLinkType;
+    resolutionStatus: SerpLinkResolutionStatus;
+}
+interface AIOverviewResult {
+    detected: boolean;
+    text: string | null;
+    citations: AICitation[];
+    expanded?: boolean;
+    fullyExpanded?: boolean;
+    sections?: string[];
+    shareUrl?: string | null;
+}
+interface AIModeResult {
+    detected: boolean;
+    text: string | null;
+    citations: AICitation[];
+}
+type GoogleSurface = 'web' | 'aim' | 'unknown';
+interface OrganicResult {
+    position: number;
+    title: string;
+    url: string;
+    rawUrl: string;
+    resolvedUrl: string | null;
+    linkType: SerpLinkType;
+    resolutionStatus: SerpLinkResolutionStatus;
+    domain: string;
+    cite: string | null;
+    snippet: string | null;
+    isRedditStyle: boolean;
+    inlineRating: {
+        value: string;
+        count: string;
+    } | null;
+}
+interface LocalPackBusiness {
+    position: number;
+    name: string;
+    cid: string | null;
+    rating: string | null;
+    reviewCount: string | null;
+    metadata: string[];
+    websiteUrl: string | null;
+    directionsUrl: string | null;
+}
+interface EntityRecord {
+    name: string;
+    kgId: string | null;
+    cid: string | null;
+    gcid: string | null;
+}
+interface EntityIds {
+    entities: EntityRecord[];
+    kgIds: string[];
+    cids: string[];
+    gcids: string[];
+}
+
 interface SerpPreviewQuestion {
     question: string;
     answer: string | null;
     url: string | null;
+    sourceTitle?: string | null;
+    sourceName?: string | null;
+    displayLink?: string | null;
 }
 interface SerpFeatureItem {
     title: string | null;
@@ -49,9 +149,9 @@ declare const HarvestOptionsSchema: z.ZodObject<{
         mobile: "mobile";
     }>>;
     proxyMode: z.ZodDefault<z.ZodEnum<{
-        none: "none";
         location: "location";
         configured: "configured";
+        none: "none";
     }>>;
     proxyZip: z.ZodOptional<z.ZodString>;
     keepDefaultProxy: z.ZodOptional<z.ZodBoolean>;
@@ -182,6 +282,8 @@ interface PaaProgressSnapshot {
 interface PaaSource extends SerpOutboundLink {
     title: string | null;
     site: string | null;
+    /** Description displayed in the Google source card, when observed. */
+    description?: string | null;
 }
 interface PAANode {
     questionId: string;
@@ -189,6 +291,8 @@ interface PAANode {
     googleEvidenceId: string | null;
     question: string;
     answer: string | null;
+    /** A remaining inner answer control could not be expanded within the capture budget. */
+    answerExpansionIncomplete?: boolean;
     sourceTitle: string | null;
     sourceSite: string | null;
     sourceCite: string | null;
@@ -204,6 +308,7 @@ interface FlatRow {
     google_evidence_id: string;
     question: string;
     answer: string;
+    answerExpansionIncomplete?: boolean;
     source_title: string;
     source_site: string;
     source_cite: string;
@@ -217,6 +322,7 @@ type HarvestOptions = z.infer<typeof HarvestOptionsSchema>;
 type CanonicalLocationName = string;
 type SerpDevice = 'desktop' | 'mobile';
 type ProxyMode = 'location' | 'configured' | 'none';
+
 interface \u004b\u0065\u0072\u006e\u0065\u006cLocationProxyTarget {
     canonicalLocation: CanonicalLocationName;
     level: 'zip' | 'city' | 'state';
@@ -244,6 +350,82 @@ interface \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionDebug {
     error: string | null;
     disposable?: boolean;
 }
+
+interface BrowserNetworkLocationDebug {
+    source: 'ipapi.co' | 'ipwho.is' | 'api64.ipify.org';
+    ip: string | null;
+    city: string | null;
+    region: string | null;
+    country: string | null;
+    org: string | null;
+    timezone: string | null;
+    error: string | null;
+}
+interface \u004b\u0065\u0072\u006e\u0065\u006cSessionDebug {
+    sessionId: string | null;
+    proxyMode: ProxyMode;
+    requestedProxyIdPresent: boolean;
+    requestedProxyIdSuffix: string | null;
+    createdProxyIdPresent: boolean | null;
+    createdProxyIdSuffix: string | null;
+    retrievedProxyIdPresent: boolean | null;
+    retrievedProxyIdSuffix: string | null;
+    retrievedProxyIdMatchesRequested: boolean | null;
+    defaultProxyDisabled: boolean | null;
+    defaultProxyDisableError: string | null;
+    proxyResolution: \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionDebug | null;
+    timeoutSeconds: number | null;
+    stealth: boolean | null;
+    profilePresent: boolean | null;
+    poolPresent: boolean | null;
+    retrieveError: string | null;
+}
+interface BrowserContextDebug {
+    viewport: {
+        width: number;
+        height: number;
+    } | null;
+    locale: string | null;
+    device: SerpDevice;
+    requestedDevice?: SerpDevice;
+    userAgent: string | null;
+    deviceScaleFactor: number | null;
+    isMobile: boolean;
+    hasTouch: boolean;
+}
+interface SerpNavigationDebug {
+    requestedUrl: string;
+    finalUrl: string | null;
+    title: string | null;
+    bodySnippet: string | null;
+    hasPaa: boolean | null;
+    captchaDetected: boolean | null;
+    googleSorryUrl: boolean | null;
+    redirected: boolean | null;
+}
+interface BrowserDebugSnapshot {
+    \u006b\u0065\u0072\u006e\u0065\u006c: \u004b\u0065\u0072\u006e\u0065\u006cSessionDebug | null;
+    context: BrowserContextDebug | null;
+    networkLocation: BrowserNetworkLocationDebug | null;
+    serpNavigation: SerpNavigationDebug | null;
+}
+
+interface SerpLocationCandidate {
+    city: string;
+    regionCode: string;
+    count: number;
+    examples: string[];
+}
+interface SerpLocationEvidence {
+    status: 'not_requested' | 'matched' | 'mismatch' | 'unknown';
+    expected: {
+        city: string;
+        regionCode: string | null;
+        canonicalLocation: string;
+    } | null;
+    candidates: SerpLocationCandidate[];
+}
+
 interface HarvestStats {
     seed: string;
     totalQuestions: number;
@@ -270,6 +452,7 @@ interface HarvestDiagnosticWarning {
 interface HarvestCompleteness {
     paaWithoutAnswer: number;
     paaWithoutSource: number;
+    paaIncompleteAnswers?: number;
     paaAnswersRecovered: number;
     aioShareCaptured: boolean | null;
     paaRequested?: number;
@@ -329,78 +512,6 @@ interface HarvestLinkDiagnostics {
     paaSourcesPreserved: number;
     aiOverviewLabelValidated: boolean;
 }
-interface BrowserNetworkLocationDebug {
-    source: 'ipapi.co' | 'ipwho.is' | 'api64.ipify.org';
-    ip: string | null;
-    city: string | null;
-    region: string | null;
-    country: string | null;
-    org: string | null;
-    timezone: string | null;
-    error: string | null;
-}
-interface \u004b\u0065\u0072\u006e\u0065\u006cSessionDebug {
-    sessionId: string | null;
-    proxyMode: ProxyMode;
-    requestedProxyIdPresent: boolean;
-    requestedProxyIdSuffix: string | null;
-    createdProxyIdPresent: boolean | null;
-    createdProxyIdSuffix: string | null;
-    retrievedProxyIdPresent: boolean | null;
-    retrievedProxyIdSuffix: string | null;
-    retrievedProxyIdMatchesRequested: boolean | null;
-    defaultProxyDisabled: boolean | null;
-    defaultProxyDisableError: string | null;
-    proxyResolution: \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionDebug | null;
-    timeoutSeconds: number | null;
-    stealth: boolean | null;
-    profilePresent: boolean | null;
-    poolPresent: boolean | null;
-    retrieveError: string | null;
-}
-interface BrowserContextDebug {
-    viewport: {
-        width: number;
-        height: number;
-    };
-    locale: string;
-    device: SerpDevice;
-    userAgent: string | null;
-    deviceScaleFactor: number | null;
-    isMobile: boolean;
-    hasTouch: boolean;
-}
-interface SerpNavigationDebug {
-    requestedUrl: string;
-    finalUrl: string | null;
-    title: string | null;
-    bodySnippet: string | null;
-    hasPaa: boolean | null;
-    captchaDetected: boolean | null;
-    googleSorryUrl: boolean | null;
-    redirected: boolean | null;
-}
-interface BrowserDebugSnapshot {
-    \u006b\u0065\u0072\u006e\u0065\u006c: \u004b\u0065\u0072\u006e\u0065\u006cSessionDebug | null;
-    context: BrowserContextDebug | null;
-    networkLocation: BrowserNetworkLocationDebug | null;
-    serpNavigation: SerpNavigationDebug | null;
-}
-interface SerpLocationCandidate {
-    city: string;
-    regionCode: string;
-    count: number;
-    examples: string[];
-}
-interface SerpLocationEvidence {
-    status: 'not_requested' | 'matched' | 'mismatch' | 'unknown';
-    expected: {
-        city: string;
-        regionCode: string | null;
-        canonicalLocation: string;
-    } | null;
-    candidates: SerpLocationCandidate[];
-}
 interface HarvestDebugSnapshot {
     enabled: true;
     request: {
@@ -423,102 +534,6 @@ interface HarvestDebugSnapshot {
     browser: BrowserDebugSnapshot;
     locationEvidence?: SerpLocationEvidence;
 }
-type SerpLinkType = 'plain' | 'google_url_redirect' | 'google_goto_redirect';
-type SerpLinkResolutionStatus = 'not_needed' | 'resolved' | 'unresolved' | 'rejected';
-interface SerpOutboundLink {
-    url: string;
-    rawUrl: string;
-    resolvedUrl: string | null;
-    linkType: SerpLinkType;
-    resolutionStatus: SerpLinkResolutionStatus;
-}
-interface VideoResult {
-    type: 'video' | 'short_video';
-    title: string;
-    channel: string;
-    platform: string;
-    duration: string;
-    url: string;
-}
-interface ForumResult {
-    title: string;
-    source: string;
-    url: string;
-}
-interface WhatPeopleSayingCard {
-    type: 'reddit' | 'facebook' | 'instagram' | 'tiktok' | 'youtube' | 'news' | 'unknown';
-    title: string;
-    url: string;
-    source: string;
-    platform: string;
-    popularComment: string | null;
-    engagement: string;
-    date: string;
-    duration: string | null;
-    authorNote: string | null;
-}
-interface AICitation {
-    text: string;
-    href: string;
-    rawUrl: string;
-    resolvedUrl: string | null;
-    linkType: SerpLinkType;
-    resolutionStatus: SerpLinkResolutionStatus;
-}
-interface AIOverviewResult {
-    detected: boolean;
-    text: string | null;
-    citations: AICitation[];
-    expanded?: boolean;
-    fullyExpanded?: boolean;
-    sections?: string[];
-    shareUrl?: string | null;
-}
-interface AIModeResult {
-    detected: boolean;
-    text: string | null;
-    citations: AICitation[];
-}
-type GoogleSurface = 'web' | 'aim' | 'unknown';
-interface OrganicResult {
-    position: number;
-    title: string;
-    url: string;
-    rawUrl: string;
-    resolvedUrl: string | null;
-    linkType: SerpLinkType;
-    resolutionStatus: SerpLinkResolutionStatus;
-    domain: string;
-    cite: string | null;
-    snippet: string | null;
-    isRedditStyle: boolean;
-    inlineRating: {
-        value: string;
-        count: string;
-    } | null;
-}
-interface LocalPackBusiness {
-    position: number;
-    name: string;
-    cid: string | null;
-    rating: string | null;
-    reviewCount: string | null;
-    metadata: string[];
-    websiteUrl: string | null;
-    directionsUrl: string | null;
-}
-interface EntityRecord {
-    name: string;
-    kgId: string | null;
-    cid: string | null;
-    gcid: string | null;
-}
-interface EntityIds {
-    entities: EntityRecord[];
-    kgIds: string[];
-    cids: string[];
-    gcids: string[];
-}
 interface HarvestResult {
     seed: string;
     location: string | null;

package/dist/index.js

@@ -1,4 +1,4 @@
-import{z}from"./chunk-MC4W6TZT.js";import"./chunk-M5QHXNFZ.js";import"./chunk-GPSPSJBJ.js";import{h as M}from"./chunk-VREXSMC6.js";import"./chunk-6DTXIZY2.js";import"./chunk-XPEJB4BZ.js";import"./chunk-7YOXSMHG.js";import{execSync as ce}from"child_process";import{readFileSync as G,writeFileSync as _,unlinkSync as D,mkdirSync as Y}from"fs";import{tmpdir as W}from"os";import{join as l}from"path";import{fal as p}from"@fal-ai/client";var X="https://api.deepinfra.com/v1/openai/chat/completions",L="https://openrouter.ai/api/v1/chat/completions",A="Qwen/Qwen3.6-35B-A3B",Z=`You are a video prompt engineer for Seedance 2.0 text-to-video AI.
+import{J as z}from"./chunk-HVQZRQRJ.js";import"./chunk-M5QHXNFZ.js";import"./chunk-B4ZKPLSP.js";import{g as M}from"./chunk-VVSHZV5S.js";import"./chunk-C7MGJUTU.js";import"./chunk-4ZVO6GX4.js";import"./chunk-WNUXMPVX.js";import"./chunk-6N5EC7ZO.js";import"./chunk-HNNYXMSI.js";import{execSync as ce}from"child_process";import{readFileSync as G,writeFileSync as _,unlinkSync as D,mkdirSync as Y}from"fs";import{tmpdir as W}from"os";import{join as l}from"path";import{fal as p}from"@fal-ai/client";var X="https://api.deepinfra.com/v1/openai/chat/completions",L="https://openrouter.ai/api/v1/chat/completions",A="Qwen/Qwen3.6-35B-A3B",Z=`You are a video prompt engineer for Seedance 2.0 text-to-video AI.
 
 IMPORTANT TECHNICAL CONTEXT: Clip 2 will be generated using image-to-video, meaning it will start from the exact last frame of Clip 1. The two clips will be visually seamless \u2014 same location, same characters, continuous motion. You must write prompts that make this feel like one uninterrupted 16-second video.
 

package/dist/media-artifacts-67CJWI7O.js

@@ -0,0 +1 @@
+import{a,b,c,d,e,f}from"./chunk-HCK6PLM3.js";import"./chunk-JMZIDHYQ.js";import"./chunk-SEZETSB3.js";import"./chunk-SDQW7IJN.js";import"./chunk-OLYPLX5C.js";import"./chunk-B3JRL45T.js";import"./chunk-OILOHQNL.js";export{a as PAGE_MEDIA_ARTIFACT_PREFIX,b as PAGE_MEDIA_ARTIFACT_TTL_MS,f as packagePageMedia,d as pageMediaArtifactOwnerId,c as pageMediaArtifactPolicy,e as readOwnedPageMediaArtifactBuffer};

package/dist/metering-ZH2P62IC.js

@@ -0,0 +1 @@
+import{a,b,c,d,e,f,g}from"./chunk-5BKSKLHX.js";import"./chunk-7SR7MEU5.js";import"./chunk-HUNSEGUN.js";import"./chunk-4ZVO6GX4.js";import"./chunk-WNUXMPVX.js";import"./chunk-HNNYXMSI.js";export{g as finishExtractUrlCostRun,b as finishMeteredProviderAction,d as linkCurrentCostDebit,f as startExtractUrlCostRun,a as startMeteredProviderAction,c as withLocalCostPhase,e as withLogicalCostRun};

package/dist/migrations-3AKEC5QE.js

@@ -0,0 +1 @@
+import{a}from"./chunk-UIOSADPB.js";import"./chunk-NNOX3FYG.js";import"./chunk-NALXKTUU.js";export{a as migrateAnalytics};

package/dist/queue-AC4XKD6Z.js

@@ -0,0 +1 @@
+import{t as d,u,v as l,w as f,x as v,xb as S,zb as h}from"./chunk-H6MODA7W.js";import"./chunk-SFUOSJ3A.js";import"./chunk-NNOX3FYG.js";import"./chunk-NALXKTUU.js";import"./chunk-6BD75VKS.js";import"./chunk-2IATUJO6.js";import"./chunk-CORWWDHL.js";import"./chunk-5LYINW7Z.js";import"./chunk-VVSHZV5S.js";import"./chunk-4ZVO6GX4.js";import"./chunk-WNUXMPVX.js";import"./chunk-WP3QS4CJ.js";var g=12;function b(e){return e&&typeof e=="object"&&"code"in e&&typeof e.code=="string"?e.code:e instanceof Error?e.name:"unknown_error"}function k(e){return Math.min(21600,30*2**Math.max(0,e-1))}async function x(e={}){let w=e.stripeClient??S,E=Math.max(1,Math.min(100,e.limit??100)),r=500,i=0,s=0,c=0,m=0;for(;i<r;){let a=await d(Math.min(E,r-i),e.leaseSeconds??300);if(a.length===0)break;i+=a.length;for(let t of a)try{let n=await w.events.retrieve(t.eventId);await h(n),await u(t.eventId,t.token)&&(s+=1)}catch(n){let p=n instanceof Error?n.message:String(n),o=b(n);t.attemptCount>=g||o==="resource_missing"?await f(t.eventId,t.token,p,o)&&(m+=1):await l(t.eventId,t.token,p,o,k(t.attemptCount))&&(c+=1)}}return{claimed:i,processed:s,retried:c,deadLettered:m,capped:i>=r,health:await v()}}export{x as drainStripeEventQueue};

package/dist/reconciliation-UZUZ6B74.js

@@ -0,0 +1 @@
+import{i as b,j as c}from"./chunk-RYX4FJVH.js";import{d as a}from"./chunk-UXNR3K6I.js";import"./chunk-BPKTUCV7.js";import"./chunk-6BD75VKS.js";import"./chunk-ZTMQEDIB.js";import"./chunk-U4WIJZCX.js";import"./chunk-7WPCGRD7.js";import"./chunk-KXJCGV3H.js";import"./chunk-JMZIDHYQ.js";import"./chunk-M5QHXNFZ.js";import"./chunk-QER2ELZP.js";import"./chunk-CORWWDHL.js";import"./chunk-5LYINW7Z.js";import"./chunk-CNSVZTTT.js";import"./chunk-5BKSKLHX.js";import"./chunk-7SR7MEU5.js";import"./chunk-HUNSEGUN.js";import"./chunk-SEZETSB3.js";import"./chunk-SDQW7IJN.js";import"./chunk-C7MGJUTU.js";import"./chunk-4ZVO6GX4.js";import"./chunk-WNUXMPVX.js";import"./chunk-WP3QS4CJ.js";import"./chunk-OLYPLX5C.js";import"./chunk-B3JRL45T.js";import"./chunk-OILOHQNL.js";import"./chunk-HNNYXMSI.js";export{c as reconcileSiteExtractSettlements,a as redispatchFundedSiteExtractJobs,b as refinalizeExtractJob};

package/dist/report-artifact-offload-6OOY2PTM.js

@@ -0,0 +1 @@
+import{a,b,c,d,e,f,g,h,i,j,k,l,m,n,o}from"./chunk-MVMWPKRC.js";import"./chunk-63VWBMYR.js";import"./chunk-3F7AFM53.js";import"./chunk-RDHGQ2WZ.js";import"./chunk-SEZETSB3.js";import"./chunk-SDQW7IJN.js";import"./chunk-C7MGJUTU.js";import"./chunk-WNUXMPVX.js";import"./chunk-OILOHQNL.js";export{h as ARTIFACT_OFFLOAD_ENABLED,c as EXTRACT_URL_JOB_ARTIFACT_PREFIX,f as MCP_RESULT_ARTIFACT_PREFIX,d as PREVIEW_CHARS,e as REPORT_ARTIFACT_POLICY,b as REPORT_BLOB_PREFIX,a as REPORT_BLOB_TTL_MS,k as artifactOwnerId,n as artifactResourceLink,l as extractUrlJobResultArtifact,g as mcpResultArtifactPolicy,j as offloadLocalExtractUrlResult,i as offloadReport,m as readArtifactWindow,o as summaryEnvelope};

package/dist/repository-4EC5EXVN.js

@@ -0,0 +1 @@
+import{a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t}from"./chunk-HUNSEGUN.js";import"./chunk-4ZVO6GX4.js";import"./chunk-WNUXMPVX.js";import"./chunk-HNNYXMSI.js";export{a as claimExecution,d as claimOperationRecovery,k as createAdmittedOperation,l as finishOperationRun,o as hasOperationExecution,p as hasUnknownOperationGroup,i as isExecutionClaimCurrent,h as markExecutionUnknown,q as noActiveOperationGroupFence,t as noUnresolvedOperationGroupFence,m as operationExecutionFence,f as publishOperationRecovery,e as publishOperationResult,n as readOperationIntentResult,s as readStoredOperationIntent,g as releaseExecutionClaim,c as renewExecutionClaim,b as startExecutionEffects,r as startGuardedExecutionEffects,j as startOperationRun};

package/dist/{location-data-repository-A2QZVP7O.js → repository-CYXVMYAI.js}

@@ -1 +1 @@
-import{a,b,c,d,e,f,g,h,i,j,k,l,m,n}from"./chunk-ZH75KWBI.js";import"./chunk-KJQXUZ4Y.js";import"./chunk-7YOXSMHG.js";export{b as HOSTED_CENSUS_DATASET_PREFIX,a as HOSTED_ZIP_DATASET_KIND,c as MIN_NATIONWIDE_ZIP_COUNT,d as REQUIRED_LOCATION_STATE_CODES,g as ensureHostedLocationDataSchema,h as getActiveHostedDataset,i as getActiveHostedLocationDataset,k as getHostedZipGroups,m as importHostedCensusPlacesCsv,l as importHostedZipGroupsCsv,j as listActiveHostedLocationDatasets,e as parseHostedCensusPlacesCsv,f as parseHostedZipGroupsCsv,n as queryHostedLocationMarkets};
+import{a,b,c,d,e,f,g,h,i,j,k,l,m,n}from"./chunk-UNEJYZ4T.js";import"./chunk-WNUXMPVX.js";import"./chunk-KJQXUZ4Y.js";export{b as HOSTED_CENSUS_DATASET_PREFIX,a as HOSTED_ZIP_DATASET_KIND,c as MIN_NATIONWIDE_ZIP_COUNT,d as REQUIRED_LOCATION_STATE_CODES,g as ensureHostedLocationDataSchema,h as getActiveHostedDataset,i as getActiveHostedLocationDataset,k as getHostedZipGroups,m as importHostedCensusPlacesCsv,l as importHostedZipGroupsCsv,j as listActiveHostedLocationDatasets,e as parseHostedCensusPlacesCsv,f as parseHostedZipGroupsCsv,n as queryHostedLocationMarkets};