mcp-scraper 0.95.1 → 0.96.0

package/dist/index.d.cts

@@ -3,6 +3,103 @@ import { z } from 'zod';
 declare const SERP_FEATURE_COMPLETENESS_STATES: readonly ["observed_present", "observed_absent", "incomplete", "not_requested", "unknown", "unsupported"];
 type SerpFeatureCompletenessState = typeof SERP_FEATURE_COMPLETENESS_STATES[number];
 
+type SerpLinkType = 'plain' | 'google_url_redirect' | 'google_goto_redirect';
+type SerpLinkResolutionStatus = 'not_needed' | 'resolved' | 'unresolved' | 'rejected';
+interface SerpOutboundLink {
+    url: string;
+    rawUrl: string;
+    resolvedUrl: string | null;
+    linkType: SerpLinkType;
+    resolutionStatus: SerpLinkResolutionStatus;
+}
+interface VideoResult {
+    type: 'video' | 'short_video';
+    title: string;
+    channel: string;
+    platform: string;
+    duration: string;
+    url: string;
+}
+interface ForumResult {
+    title: string;
+    source: string;
+    url: string;
+}
+interface WhatPeopleSayingCard {
+    type: 'reddit' | 'facebook' | 'instagram' | 'tiktok' | 'youtube' | 'news' | 'unknown';
+    title: string;
+    url: string;
+    source: string;
+    platform: string;
+    popularComment: string | null;
+    engagement: string;
+    date: string;
+    duration: string | null;
+    authorNote: string | null;
+}
+interface AICitation {
+    text: string;
+    href: string;
+    rawUrl: string;
+    resolvedUrl: string | null;
+    linkType: SerpLinkType;
+    resolutionStatus: SerpLinkResolutionStatus;
+}
+interface AIOverviewResult {
+    detected: boolean;
+    text: string | null;
+    citations: AICitation[];
+    expanded?: boolean;
+    fullyExpanded?: boolean;
+    sections?: string[];
+    shareUrl?: string | null;
+}
+interface AIModeResult {
+    detected: boolean;
+    text: string | null;
+    citations: AICitation[];
+}
+type GoogleSurface = 'web' | 'aim' | 'unknown';
+interface OrganicResult {
+    position: number;
+    title: string;
+    url: string;
+    rawUrl: string;
+    resolvedUrl: string | null;
+    linkType: SerpLinkType;
+    resolutionStatus: SerpLinkResolutionStatus;
+    domain: string;
+    cite: string | null;
+    snippet: string | null;
+    isRedditStyle: boolean;
+    inlineRating: {
+        value: string;
+        count: string;
+    } | null;
+}
+interface LocalPackBusiness {
+    position: number;
+    name: string;
+    cid: string | null;
+    rating: string | null;
+    reviewCount: string | null;
+    metadata: string[];
+    websiteUrl: string | null;
+    directionsUrl: string | null;
+}
+interface EntityRecord {
+    name: string;
+    kgId: string | null;
+    cid: string | null;
+    gcid: string | null;
+}
+interface EntityIds {
+    entities: EntityRecord[];
+    kgIds: string[];
+    cids: string[];
+    gcids: string[];
+}
+
 interface SerpPreviewQuestion {
     question: string;
     answer: string | null;
@@ -49,9 +146,9 @@ declare const HarvestOptionsSchema: z.ZodObject<{
         mobile: "mobile";
     }>>;
     proxyMode: z.ZodDefault<z.ZodEnum<{
-        none: "none";
         location: "location";
         configured: "configured";
+        none: "none";
     }>>;
     proxyZip: z.ZodOptional<z.ZodString>;
     keepDefaultProxy: z.ZodOptional<z.ZodBoolean>;
@@ -63,13 +160,13 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     headless: z.ZodDefault<z.ZodBoolean>;
     profileDir: z.ZodOptional<z.ZodString>;
     proxy: z.ZodOptional<z.ZodString>;
-    kernelApiKey: z.ZodOptional<z.ZodString>;
-    kernelProxyId: z.ZodOptional<z.ZodString>;
-    kernelProfileName: z.ZodOptional<z.ZodString>;
-    kernelProfileSaveChanges: z.ZodOptional<z.ZodBoolean>;
-    kernelStealth: z.ZodOptional<z.ZodBoolean>;
+    \u006b\u0065\u0072\u006e\u0065\u006cApiKey: z.ZodOptional<z.ZodString>;
+    \u006b\u0065\u0072\u006e\u0065\u006cProxyId: z.ZodOptional<z.ZodString>;
+    \u006b\u0065\u0072\u006e\u0065\u006cProfileName: z.ZodOptional<z.ZodString>;
+    \u006b\u0065\u0072\u006e\u0065\u006cProfileSaveChanges: z.ZodOptional<z.ZodBoolean>;
+    \u006b\u0065\u0072\u006e\u0065\u006cStealth: z.ZodOptional<z.ZodBoolean>;
     serpIdentity: z.ZodOptional<z.ZodString>;
-    kernelProxyResolution: z.ZodOptional<z.ZodUnknown>;
+    \u006b\u0065\u0072\u006e\u0065\u006cProxyResolution: z.ZodOptional<z.ZodUnknown>;
     outputDir: z.ZodDefault<z.ZodString>;
     format: z.ZodDefault<z.ZodEnum<{
         json: "json";
@@ -182,6 +279,8 @@ interface PaaProgressSnapshot {
 interface PaaSource extends SerpOutboundLink {
     title: string | null;
     site: string | null;
+    /** Description displayed in the Google source card, when observed. */
+    description?: string | null;
 }
 interface PAANode {
     questionId: string;
@@ -189,6 +288,8 @@ interface PAANode {
     googleEvidenceId: string | null;
     question: string;
     answer: string | null;
+    /** A remaining inner answer control could not be expanded within the capture budget. */
+    answerExpansionIncomplete?: boolean;
     sourceTitle: string | null;
     sourceSite: string | null;
     sourceCite: string | null;
@@ -204,6 +305,7 @@ interface FlatRow {
     google_evidence_id: string;
     question: string;
     answer: string;
+    answerExpansionIncomplete?: boolean;
     source_title: string;
     source_site: string;
     source_cite: string;
@@ -217,7 +319,8 @@ type HarvestOptions = z.infer<typeof HarvestOptionsSchema>;
 type CanonicalLocationName = string;
 type SerpDevice = 'desktop' | 'mobile';
 type ProxyMode = 'location' | 'configured' | 'none';
-interface KernelLocationProxyTarget {
+
+interface \u004b\u0065\u0072\u006e\u0065\u006cLocationProxyTarget {
     canonicalLocation: CanonicalLocationName;
     level: 'zip' | 'city' | 'state';
     country: 'US';
@@ -233,17 +336,93 @@ interface KernelLocationProxyTarget {
         zip?: string;
     };
 }
-type KernelProxyResolutionSource = 'disabled' | 'location_reused' | 'location_created' | 'mobile_created' | 'isp_created' | 'backup_custom' | 'configured_fallback' | 'unavailable';
-interface KernelProxyResolutionDebug {
-    source: KernelProxyResolutionSource;
+type \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionSource = 'disabled' | 'location_reused' | 'location_created' | 'mobile_created' | 'isp_created' | 'backup_custom' | 'configured_fallback' | 'unavailable';
+interface \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionDebug {
+    source: \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionSource;
     proxyType: string;
     proxyMode: ProxyMode;
     proxyIdPresent: boolean;
     proxyIdSuffix: string | null;
-    target: KernelLocationProxyTarget | null;
+    target: \u004b\u0065\u0072\u006e\u0065\u006cLocationProxyTarget | null;
     error: string | null;
     disposable?: boolean;
 }
+
+interface BrowserNetworkLocationDebug {
+    source: 'ipapi.co' | 'ipwho.is' | 'api64.ipify.org';
+    ip: string | null;
+    city: string | null;
+    region: string | null;
+    country: string | null;
+    org: string | null;
+    timezone: string | null;
+    error: string | null;
+}
+interface \u004b\u0065\u0072\u006e\u0065\u006cSessionDebug {
+    sessionId: string | null;
+    proxyMode: ProxyMode;
+    requestedProxyIdPresent: boolean;
+    requestedProxyIdSuffix: string | null;
+    createdProxyIdPresent: boolean | null;
+    createdProxyIdSuffix: string | null;
+    retrievedProxyIdPresent: boolean | null;
+    retrievedProxyIdSuffix: string | null;
+    retrievedProxyIdMatchesRequested: boolean | null;
+    defaultProxyDisabled: boolean | null;
+    defaultProxyDisableError: string | null;
+    proxyResolution: \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionDebug | null;
+    timeoutSeconds: number | null;
+    stealth: boolean | null;
+    profilePresent: boolean | null;
+    poolPresent: boolean | null;
+    retrieveError: string | null;
+}
+interface BrowserContextDebug {
+    viewport: {
+        width: number;
+        height: number;
+    } | null;
+    locale: string | null;
+    device: SerpDevice;
+    requestedDevice?: SerpDevice;
+    userAgent: string | null;
+    deviceScaleFactor: number | null;
+    isMobile: boolean;
+    hasTouch: boolean;
+}
+interface SerpNavigationDebug {
+    requestedUrl: string;
+    finalUrl: string | null;
+    title: string | null;
+    bodySnippet: string | null;
+    hasPaa: boolean | null;
+    captchaDetected: boolean | null;
+    googleSorryUrl: boolean | null;
+    redirected: boolean | null;
+}
+interface BrowserDebugSnapshot {
+    \u006b\u0065\u0072\u006e\u0065\u006c: \u004b\u0065\u0072\u006e\u0065\u006cSessionDebug | null;
+    context: BrowserContextDebug | null;
+    networkLocation: BrowserNetworkLocationDebug | null;
+    serpNavigation: SerpNavigationDebug | null;
+}
+
+interface SerpLocationCandidate {
+    city: string;
+    regionCode: string;
+    count: number;
+    examples: string[];
+}
+interface SerpLocationEvidence {
+    status: 'not_requested' | 'matched' | 'mismatch' | 'unknown';
+    expected: {
+        city: string;
+        regionCode: string | null;
+        canonicalLocation: string;
+    } | null;
+    candidates: SerpLocationCandidate[];
+}
+
 interface HarvestStats {
     seed: string;
     totalQuestions: number;
@@ -270,6 +449,7 @@ interface HarvestDiagnosticWarning {
 interface HarvestCompleteness {
     paaWithoutAnswer: number;
     paaWithoutSource: number;
+    paaIncompleteAnswers?: number;
     paaAnswersRecovered: number;
     aioShareCaptured: boolean | null;
     paaRequested?: number;
@@ -329,78 +509,6 @@ interface HarvestLinkDiagnostics {
     paaSourcesPreserved: number;
     aiOverviewLabelValidated: boolean;
 }
-interface BrowserNetworkLocationDebug {
-    source: 'ipapi.co' | 'ipwho.is' | 'api64.ipify.org';
-    ip: string | null;
-    city: string | null;
-    region: string | null;
-    country: string | null;
-    org: string | null;
-    timezone: string | null;
-    error: string | null;
-}
-interface KernelSessionDebug {
-    sessionId: string | null;
-    proxyMode: ProxyMode;
-    requestedProxyIdPresent: boolean;
-    requestedProxyIdSuffix: string | null;
-    createdProxyIdPresent: boolean | null;
-    createdProxyIdSuffix: string | null;
-    retrievedProxyIdPresent: boolean | null;
-    retrievedProxyIdSuffix: string | null;
-    retrievedProxyIdMatchesRequested: boolean | null;
-    defaultProxyDisabled: boolean | null;
-    defaultProxyDisableError: string | null;
-    proxyResolution: KernelProxyResolutionDebug | null;
-    timeoutSeconds: number | null;
-    stealth: boolean | null;
-    profilePresent: boolean | null;
-    poolPresent: boolean | null;
-    retrieveError: string | null;
-}
-interface BrowserContextDebug {
-    viewport: {
-        width: number;
-        height: number;
-    };
-    locale: string;
-    device: SerpDevice;
-    userAgent: string | null;
-    deviceScaleFactor: number | null;
-    isMobile: boolean;
-    hasTouch: boolean;
-}
-interface SerpNavigationDebug {
-    requestedUrl: string;
-    finalUrl: string | null;
-    title: string | null;
-    bodySnippet: string | null;
-    hasPaa: boolean | null;
-    captchaDetected: boolean | null;
-    googleSorryUrl: boolean | null;
-    redirected: boolean | null;
-}
-interface BrowserDebugSnapshot {
-    kernel: KernelSessionDebug | null;
-    context: BrowserContextDebug | null;
-    networkLocation: BrowserNetworkLocationDebug | null;
-    serpNavigation: SerpNavigationDebug | null;
-}
-interface SerpLocationCandidate {
-    city: string;
-    regionCode: string;
-    count: number;
-    examples: string[];
-}
-interface SerpLocationEvidence {
-    status: 'not_requested' | 'matched' | 'mismatch' | 'unknown';
-    expected: {
-        city: string;
-        regionCode: string | null;
-        canonicalLocation: string;
-    } | null;
-    candidates: SerpLocationCandidate[];
-}
 interface HarvestDebugSnapshot {
     enabled: true;
     request: {
@@ -423,102 +531,6 @@ interface HarvestDebugSnapshot {
     browser: BrowserDebugSnapshot;
     locationEvidence?: SerpLocationEvidence;
 }
-type SerpLinkType = 'plain' | 'google_url_redirect' | 'google_goto_redirect';
-type SerpLinkResolutionStatus = 'not_needed' | 'resolved' | 'unresolved' | 'rejected';
-interface SerpOutboundLink {
-    url: string;
-    rawUrl: string;
-    resolvedUrl: string | null;
-    linkType: SerpLinkType;
-    resolutionStatus: SerpLinkResolutionStatus;
-}
-interface VideoResult {
-    type: 'video' | 'short_video';
-    title: string;
-    channel: string;
-    platform: string;
-    duration: string;
-    url: string;
-}
-interface ForumResult {
-    title: string;
-    source: string;
-    url: string;
-}
-interface WhatPeopleSayingCard {
-    type: 'reddit' | 'facebook' | 'instagram' | 'tiktok' | 'youtube' | 'news' | 'unknown';
-    title: string;
-    url: string;
-    source: string;
-    platform: string;
-    popularComment: string | null;
-    engagement: string;
-    date: string;
-    duration: string | null;
-    authorNote: string | null;
-}
-interface AICitation {
-    text: string;
-    href: string;
-    rawUrl: string;
-    resolvedUrl: string | null;
-    linkType: SerpLinkType;
-    resolutionStatus: SerpLinkResolutionStatus;
-}
-interface AIOverviewResult {
-    detected: boolean;
-    text: string | null;
-    citations: AICitation[];
-    expanded?: boolean;
-    fullyExpanded?: boolean;
-    sections?: string[];
-    shareUrl?: string | null;
-}
-interface AIModeResult {
-    detected: boolean;
-    text: string | null;
-    citations: AICitation[];
-}
-type GoogleSurface = 'web' | 'aim' | 'unknown';
-interface OrganicResult {
-    position: number;
-    title: string;
-    url: string;
-    rawUrl: string;
-    resolvedUrl: string | null;
-    linkType: SerpLinkType;
-    resolutionStatus: SerpLinkResolutionStatus;
-    domain: string;
-    cite: string | null;
-    snippet: string | null;
-    isRedditStyle: boolean;
-    inlineRating: {
-        value: string;
-        count: string;
-    } | null;
-}
-interface LocalPackBusiness {
-    position: number;
-    name: string;
-    cid: string | null;
-    rating: string | null;
-    reviewCount: string | null;
-    metadata: string[];
-    websiteUrl: string | null;
-    directionsUrl: string | null;
-}
-interface EntityRecord {
-    name: string;
-    kgId: string | null;
-    cid: string | null;
-    gcid: string | null;
-}
-interface EntityIds {
-    entities: EntityRecord[];
-    kgIds: string[];
-    cids: string[];
-    gcids: string[];
-}
 interface HarvestResult {
     seed: string;
     location: string | null;