mcp-scraper 0.76.0 → 0.78.0

package/dist/index.d.cts

@@ -126,6 +126,45 @@ interface HarvestInterruptionDiagnostic {
     reasonSource: HarvestInterruptionReasonSource;
     message: string | null;
 }
+type PaaDiscoveryStatus = 'in_progress' | 'target_reached' | 'frontier_exhausted' | 'no_paa_observed' | 'interrupted' | 'recovery_exhausted';
+type PaaTargetStatus = 'met' | 'not_met';
+type PaaMaterialStatus = 'complete' | 'partial';
+interface PaaLifecycleDiagnostics {
+    requestedQuestions: number;
+    questionsFound: number;
+    targetStatus: PaaTargetStatus;
+    discoveryStatus: PaaDiscoveryStatus;
+    materialStatus: PaaMaterialStatus;
+    completionReason: string;
+    automaticRetries: number;
+    knownQuestions: number;
+    processedQuestions: number;
+    failedInteractions: number;
+    noGrowthConfirmations: number;
+}
+type PaaProgressPhase = 'serp_captured' | 'discovered' | 'click_observed' | 'answer_captured' | 'fill_incomplete' | 'expansion_finished' | 'attempt_ending';
+/** Material already observed before PAA expansion completes. It is checkpointed
+ * so a worker interruption cannot discard the SERP that led to the PAA graph. */
+interface PaaProgressMaterial {
+    surface: GoogleSurface;
+    aiOverview: AIOverviewResult;
+    aiMode: AIModeResult;
+    whatPeopleSaying: WhatPeopleSayingCard[];
+    videos: VideoResult[];
+    forums: ForumResult[];
+    organicResults: OrganicResult[];
+    localPack: LocalPackBusiness[];
+    entityIds: EntityIds;
+}
+interface PaaProgressSnapshot {
+    sequence: number;
+    observedAt: string;
+    clickedQuestions: string[];
+    phase: PaaProgressPhase;
+    records: FlatRow[];
+    lifecycle?: PaaLifecycleDiagnostics;
+    material?: PaaProgressMaterial;
+}
 interface HarvestDiagnostics {
     completionStatus: HarvestCompletionStatus;
     /** Set only when the extractor directly observed a readable SERP without a PAA block. */
@@ -137,9 +176,23 @@ interface HarvestDiagnostics {
     retryRecommended?: boolean;
     interruption?: HarvestInterruptionDiagnostic;
     warnings?: HarvestDiagnosticWarning[];
+    links?: HarvestLinkDiagnostics;
     completeness?: HarvestCompleteness;
+    paaLifecycle?: PaaLifecycleDiagnostics;
     debug?: HarvestDebugSnapshot;
 }
+interface HarvestLinkDiagnostics {
+    linkCandidatesObserved: number;
+    plainLinksObserved: number;
+    googleUrlRedirectsObserved: number;
+    googleGotoRedirectsObserved: number;
+    redirectsResolved: number;
+    redirectsUnresolved: number;
+    redirectsRejected: number;
+    resolverNonGoogleRequestsObserved: number;
+    paaSourcesPreserved: number;
+    aiOverviewLabelValidated: boolean;
+}
 interface BrowserNetworkLocationDebug {
     source: 'ipapi.co' | 'ipwho.is' | 'api64.ipify.org';
     ip: string | null;
@@ -234,23 +287,44 @@ interface HarvestDebugSnapshot {
     browser: BrowserDebugSnapshot;
     locationEvidence?: SerpLocationEvidence;
 }
+type SerpLinkType = 'plain' | 'google_url_redirect' | 'google_goto_redirect';
+type SerpLinkResolutionStatus = 'not_needed' | 'resolved' | 'unresolved' | 'rejected';
+interface SerpOutboundLink {
+    url: string;
+    rawUrl: string;
+    resolvedUrl: string | null;
+    linkType: SerpLinkType;
+    resolutionStatus: SerpLinkResolutionStatus;
+}
+interface PaaSource extends SerpOutboundLink {
+    title: string | null;
+    site: string | null;
+}
 interface PAANode {
+    questionId: string;
+    googleLinkId: string | null;
+    googleEvidenceId: string | null;
     question: string;
     answer: string | null;
     sourceTitle: string | null;
     sourceSite: string | null;
     sourceCite: string | null;
+    sources: PaaSource[];
     depth: number;
     parentQuestion: string | null;
     children: PAANode[];
 }
 interface FlatRow {
     seed_query: string;
+    question_id: string;
+    google_link_id: string;
+    google_evidence_id: string;
     question: string;
     answer: string;
     source_title: string;
     source_site: string;
     source_cite: string;
+    sources: PaaSource[];
     depth: number;
     parent_question: string;
     extracted_at: string;
@@ -283,6 +357,10 @@ interface WhatPeopleSayingCard {
 interface AICitation {
     text: string;
     href: string;
+    rawUrl: string;
+    resolvedUrl: string | null;
+    linkType: SerpLinkType;
+    resolutionStatus: SerpLinkResolutionStatus;
 }
 interface AIOverviewResult {
     detected: boolean;
@@ -303,6 +381,10 @@ interface OrganicResult {
     position: number;
     title: string;
     url: string;
+    rawUrl: string;
+    resolvedUrl: string | null;
+    linkType: SerpLinkType;
+    resolutionStatus: SerpLinkResolutionStatus;
     domain: string;
     cite: string | null;
     snippet: string | null;
@@ -404,4 +486,4 @@ interface ClipPromptPair {
 }
 declare function buildClipPrompts(question: string, answer: string): Promise<ClipPromptPair>;
 
-export { type AICitation, type AIModeResult, type AIOverviewResult, type ClipPairOptions, type ClipPairResult, type FlatRow, type GoogleSurface, type HarvestCompletionStatus, type HarvestDiagnostics, type HarvestOptions, type HarvestResult, type HarvestStats, type PAANode, VideoGenerator, type WhatPeopleSayingCard, buildClipPrompts, harvest };
+export { type AICitation, type AIModeResult, type AIOverviewResult, type ClipPairOptions, type ClipPairResult, type FlatRow, type GoogleSurface, type HarvestCompletionStatus, type HarvestDiagnostics, type HarvestOptions, type HarvestResult, type HarvestStats, type PAANode, type PaaDiscoveryStatus, type PaaLifecycleDiagnostics, type PaaMaterialStatus, type PaaProgressMaterial, type PaaProgressSnapshot, type PaaTargetStatus, VideoGenerator, type WhatPeopleSayingCard, buildClipPrompts, harvest };

package/dist/index.d.ts

@@ -126,6 +126,45 @@ interface HarvestInterruptionDiagnostic {
     reasonSource: HarvestInterruptionReasonSource;
     message: string | null;
 }
+type PaaDiscoveryStatus = 'in_progress' | 'target_reached' | 'frontier_exhausted' | 'no_paa_observed' | 'interrupted' | 'recovery_exhausted';
+type PaaTargetStatus = 'met' | 'not_met';
+type PaaMaterialStatus = 'complete' | 'partial';
+interface PaaLifecycleDiagnostics {
+    requestedQuestions: number;
+    questionsFound: number;
+    targetStatus: PaaTargetStatus;
+    discoveryStatus: PaaDiscoveryStatus;
+    materialStatus: PaaMaterialStatus;
+    completionReason: string;
+    automaticRetries: number;
+    knownQuestions: number;
+    processedQuestions: number;
+    failedInteractions: number;
+    noGrowthConfirmations: number;
+}
+type PaaProgressPhase = 'serp_captured' | 'discovered' | 'click_observed' | 'answer_captured' | 'fill_incomplete' | 'expansion_finished' | 'attempt_ending';
+/** Material already observed before PAA expansion completes. It is checkpointed
+ * so a worker interruption cannot discard the SERP that led to the PAA graph. */
+interface PaaProgressMaterial {
+    surface: GoogleSurface;
+    aiOverview: AIOverviewResult;
+    aiMode: AIModeResult;
+    whatPeopleSaying: WhatPeopleSayingCard[];
+    videos: VideoResult[];
+    forums: ForumResult[];
+    organicResults: OrganicResult[];
+    localPack: LocalPackBusiness[];
+    entityIds: EntityIds;
+}
+interface PaaProgressSnapshot {
+    sequence: number;
+    observedAt: string;
+    clickedQuestions: string[];
+    phase: PaaProgressPhase;
+    records: FlatRow[];
+    lifecycle?: PaaLifecycleDiagnostics;
+    material?: PaaProgressMaterial;
+}
 interface HarvestDiagnostics {
     completionStatus: HarvestCompletionStatus;
     /** Set only when the extractor directly observed a readable SERP without a PAA block. */
@@ -137,9 +176,23 @@ interface HarvestDiagnostics {
     retryRecommended?: boolean;
     interruption?: HarvestInterruptionDiagnostic;
     warnings?: HarvestDiagnosticWarning[];
+    links?: HarvestLinkDiagnostics;
     completeness?: HarvestCompleteness;
+    paaLifecycle?: PaaLifecycleDiagnostics;
     debug?: HarvestDebugSnapshot;
 }
+interface HarvestLinkDiagnostics {
+    linkCandidatesObserved: number;
+    plainLinksObserved: number;
+    googleUrlRedirectsObserved: number;
+    googleGotoRedirectsObserved: number;
+    redirectsResolved: number;
+    redirectsUnresolved: number;
+    redirectsRejected: number;
+    resolverNonGoogleRequestsObserved: number;
+    paaSourcesPreserved: number;
+    aiOverviewLabelValidated: boolean;
+}
 interface BrowserNetworkLocationDebug {
     source: 'ipapi.co' | 'ipwho.is' | 'api64.ipify.org';
     ip: string | null;
@@ -234,23 +287,44 @@ interface HarvestDebugSnapshot {
     browser: BrowserDebugSnapshot;
     locationEvidence?: SerpLocationEvidence;
 }
+type SerpLinkType = 'plain' | 'google_url_redirect' | 'google_goto_redirect';
+type SerpLinkResolutionStatus = 'not_needed' | 'resolved' | 'unresolved' | 'rejected';
+interface SerpOutboundLink {
+    url: string;
+    rawUrl: string;
+    resolvedUrl: string | null;
+    linkType: SerpLinkType;
+    resolutionStatus: SerpLinkResolutionStatus;
+}
+interface PaaSource extends SerpOutboundLink {
+    title: string | null;
+    site: string | null;
+}
 interface PAANode {
+    questionId: string;
+    googleLinkId: string | null;
+    googleEvidenceId: string | null;
     question: string;
     answer: string | null;
     sourceTitle: string | null;
     sourceSite: string | null;
     sourceCite: string | null;
+    sources: PaaSource[];
     depth: number;
     parentQuestion: string | null;
     children: PAANode[];
 }
 interface FlatRow {
     seed_query: string;
+    question_id: string;
+    google_link_id: string;
+    google_evidence_id: string;
     question: string;
     answer: string;
     source_title: string;
     source_site: string;
     source_cite: string;
+    sources: PaaSource[];
     depth: number;
     parent_question: string;
     extracted_at: string;
@@ -283,6 +357,10 @@ interface WhatPeopleSayingCard {
 interface AICitation {
     text: string;
     href: string;
+    rawUrl: string;
+    resolvedUrl: string | null;
+    linkType: SerpLinkType;
+    resolutionStatus: SerpLinkResolutionStatus;
 }
 interface AIOverviewResult {
     detected: boolean;
@@ -303,6 +381,10 @@ interface OrganicResult {
     position: number;
     title: string;
     url: string;
+    rawUrl: string;
+    resolvedUrl: string | null;
+    linkType: SerpLinkType;
+    resolutionStatus: SerpLinkResolutionStatus;
     domain: string;
     cite: string | null;
     snippet: string | null;
@@ -404,4 +486,4 @@ interface ClipPromptPair {
 }
 declare function buildClipPrompts(question: string, answer: string): Promise<ClipPromptPair>;
 
-export { type AICitation, type AIModeResult, type AIOverviewResult, type ClipPairOptions, type ClipPairResult, type FlatRow, type GoogleSurface, type HarvestCompletionStatus, type HarvestDiagnostics, type HarvestOptions, type HarvestResult, type HarvestStats, type PAANode, VideoGenerator, type WhatPeopleSayingCard, buildClipPrompts, harvest };
+export { type AICitation, type AIModeResult, type AIOverviewResult, type ClipPairOptions, type ClipPairResult, type FlatRow, type GoogleSurface, type HarvestCompletionStatus, type HarvestDiagnostics, type HarvestOptions, type HarvestResult, type HarvestStats, type PAANode, type PaaDiscoveryStatus, type PaaLifecycleDiagnostics, type PaaMaterialStatus, type PaaProgressMaterial, type PaaProgressSnapshot, type PaaTargetStatus, VideoGenerator, type WhatPeopleSayingCard, buildClipPrompts, harvest };

package/dist/index.js

@@ -1,12 +1,12 @@
 import {
   harvest
-} from "./chunk-F6MUGMRN.js";
+} from "./chunk-UFJJGXVO.js";
 import "./chunk-OM7HVEJ3.js";
 import {
   recordVendorUsage
-} from "./chunk-CXY5WV45.js";
-import "./chunk-77LQV73B.js";
-import "./chunk-WEFPBAAG.js";
+} from "./chunk-NGLZ6UJR.js";
+import "./chunk-OWF2JJKN.js";
+import "./chunk-OU4FV2RP.js";
 
 // src/video/VideoGenerator.ts
 import { execSync as execSync2 } from "child_process";

package/dist/{lead-list-enrichment-repository-WWEMJ7EN.js → lead-list-enrichment-repository-5LEWW4X7.js}

@@ -30,8 +30,8 @@ import {
   saveLeadListEnrichmentRowResult,
   setInitializingLeadListEnrichmentConcurrencyLock,
   stableCanonicalJson
-} from "./chunk-A66DGFOU.js";
-import "./chunk-WEFPBAAG.js";
+} from "./chunk-KAGHJQFO.js";
+import "./chunk-OU4FV2RP.js";
 export {
   LEAD_LIST_MAX_DISPATCH_ATTEMPTS,
   LEAD_LIST_RUNNING_STALE_MINUTES,

package/dist/{location-data-repository-H4B2BWGY.js → location-data-repository-AHKDXTEQ.js}

@@ -13,9 +13,9 @@ import {
   parseHostedCensusPlacesCsv,
   parseHostedZipGroupsCsv,
   queryHostedLocationMarkets
-} from "./chunk-F5TK4KMT.js";
+} from "./chunk-OUBH6XMY.js";
 import "./chunk-RJ7JVYKU.js";
-import "./chunk-WEFPBAAG.js";
+import "./chunk-OU4FV2RP.js";
 export {
   HOSTED_CENSUS_DATASET_PREFIX,
   HOSTED_ZIP_DATASET_KIND,