mcp-scraper 0.40.1 → 0.40.2

package/dist/index.d.ts

@@ -0,0 +1,413 @@
+import { z } from 'zod';
+
+declare const HarvestOptionsSchema: z.ZodObject<{
+    query: z.ZodString;
+    location: z.ZodOptional<z.ZodString>;
+    gl: z.ZodDefault<z.ZodString>;
+    hl: z.ZodDefault<z.ZodString>;
+    device: z.ZodDefault<z.ZodEnum<["desktop", "mobile"]>>;
+    proxyMode: z.ZodDefault<z.ZodEnum<["location", "configured", "none"]>>;
+    proxyZip: z.ZodOptional<z.ZodString>;
+    keepDefaultProxy: z.ZodOptional<z.ZodBoolean>;
+    requireUsEgress: z.ZodOptional<z.ZodBoolean>;
+    maxAttempts: z.ZodOptional<z.ZodNumber>;
+    debug: z.ZodDefault<z.ZodBoolean>;
+    depth: z.ZodDefault<z.ZodNumber>;
+    maxQuestions: z.ZodDefault<z.ZodNumber>;
+    headless: z.ZodDefault<z.ZodBoolean>;
+    profileDir: z.ZodOptional<z.ZodString>;
+    proxy: z.ZodOptional<z.ZodString>;
+    kernelApiKey: z.ZodOptional<z.ZodString>;
+    kernelProxyId: z.ZodOptional<z.ZodString>;
+    kernelProxyResolution: z.ZodOptional<z.ZodUnknown>;
+    outputDir: z.ZodDefault<z.ZodString>;
+    format: z.ZodDefault<z.ZodEnum<["json", "csv", "both"]>>;
+    serpOnly: z.ZodDefault<z.ZodBoolean>;
+    pages: z.ZodDefault<z.ZodNumber>;
+    recency: z.ZodOptional<z.ZodEnum<["day", "week", "month", "year"]>>;
+    softDeadlineMs: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    query: string;
+    gl: string;
+    hl: string;
+    device: "desktop" | "mobile";
+    proxyMode: "none" | "location" | "configured";
+    debug: boolean;
+    depth: number;
+    maxQuestions: number;
+    headless: boolean;
+    outputDir: string;
+    format: "json" | "csv" | "both";
+    serpOnly: boolean;
+    pages: number;
+    location?: string | undefined;
+    proxyZip?: string | undefined;
+    keepDefaultProxy?: boolean | undefined;
+    requireUsEgress?: boolean | undefined;
+    maxAttempts?: number | undefined;
+    profileDir?: string | undefined;
+    proxy?: string | undefined;
+    kernelApiKey?: string | undefined;
+    kernelProxyId?: string | undefined;
+    kernelProxyResolution?: unknown;
+    recency?: "day" | "week" | "month" | "year" | undefined;
+    softDeadlineMs?: number | undefined;
+}, {
+    query: string;
+    location?: string | undefined;
+    gl?: string | undefined;
+    hl?: string | undefined;
+    device?: "desktop" | "mobile" | undefined;
+    proxyMode?: "none" | "location" | "configured" | undefined;
+    proxyZip?: string | undefined;
+    keepDefaultProxy?: boolean | undefined;
+    requireUsEgress?: boolean | undefined;
+    maxAttempts?: number | undefined;
+    debug?: boolean | undefined;
+    depth?: number | undefined;
+    maxQuestions?: number | undefined;
+    headless?: boolean | undefined;
+    profileDir?: string | undefined;
+    proxy?: string | undefined;
+    kernelApiKey?: string | undefined;
+    kernelProxyId?: string | undefined;
+    kernelProxyResolution?: unknown;
+    outputDir?: string | undefined;
+    format?: "json" | "csv" | "both" | undefined;
+    serpOnly?: boolean | undefined;
+    pages?: number | undefined;
+    recency?: "day" | "week" | "month" | "year" | undefined;
+    softDeadlineMs?: number | undefined;
+}>;
+
+type CanonicalLocationName = string;
+type SerpDevice = 'desktop' | 'mobile';
+type ProxyMode = 'location' | 'configured' | 'none';
+interface KernelLocationProxyTarget {
+    canonicalLocation: CanonicalLocationName;
+    level: 'zip' | 'city' | 'state';
+    country: 'US';
+    state: string;
+    city: string;
+    zip?: string;
+    cityCandidates: string[];
+    proxyName: string;
+    config: {
+        country: 'US';
+        state: string;
+        city?: string;
+        zip?: string;
+    };
+}
+type KernelProxyResolutionSource = 'disabled' | 'location_reused' | 'location_created' | 'isp_created' | 'backup_custom' | 'configured_fallback' | 'unavailable';
+interface KernelProxyResolutionDebug {
+    source: KernelProxyResolutionSource;
+    proxyType: string;
+    proxyMode: ProxyMode;
+    proxyIdPresent: boolean;
+    proxyIdSuffix: string | null;
+    target: KernelLocationProxyTarget | null;
+    error: string | null;
+    disposable?: boolean;
+}
+interface HarvestStats {
+    seed: string;
+    totalQuestions: number;
+    maxDepthReached: number;
+    durationMs: number;
+    errorCount: number;
+}
+type HarvestCompletionStatus = 'paa_found' | 'no_paa' | 'serp_only';
+/**
+ * A completed browser attempt can still be unsuitable for programmatic use.
+ * This is deliberately separate from completionStatus so a valid no-PAA
+ * response with organic results is not confused with an unreadable SERP.
+ */
+type HarvestResultQuality = 'complete' | 'partial' | 'degraded';
+type HarvestDegradationReason = 'empty_primary_serp';
+type HarvestDiagnosticWarningCode = 'short_videos_captcha_skipped';
+type HarvestDiagnosticWarningSurface = 'short_videos';
+interface HarvestDiagnosticWarning {
+    code: HarvestDiagnosticWarningCode;
+    surface: HarvestDiagnosticWarningSurface;
+    message: string;
+    retryable: boolean;
+}
+interface HarvestCompleteness {
+    paaWithoutAnswer: number;
+    paaWithoutSource: number;
+    paaAnswersRecovered: number;
+    aioShareCaptured: boolean | null;
+}
+interface HarvestDiagnostics {
+    completionStatus: HarvestCompletionStatus;
+    problem: null;
+    resultQuality?: HarvestResultQuality;
+    degradedResult?: boolean;
+    degradationReasons?: HarvestDegradationReason[];
+    retryRecommended?: boolean;
+    warnings?: HarvestDiagnosticWarning[];
+    completeness?: HarvestCompleteness;
+    debug?: HarvestDebugSnapshot;
+}
+interface BrowserNetworkLocationDebug {
+    source: 'ipapi.co' | 'ipwho.is' | 'api64.ipify.org';
+    ip: string | null;
+    city: string | null;
+    region: string | null;
+    country: string | null;
+    org: string | null;
+    timezone: string | null;
+    error: string | null;
+}
+interface KernelSessionDebug {
+    sessionId: string | null;
+    proxyMode: ProxyMode;
+    requestedProxyIdPresent: boolean;
+    requestedProxyIdSuffix: string | null;
+    createdProxyIdPresent: boolean | null;
+    createdProxyIdSuffix: string | null;
+    retrievedProxyIdPresent: boolean | null;
+    retrievedProxyIdSuffix: string | null;
+    retrievedProxyIdMatchesRequested: boolean | null;
+    defaultProxyDisabled: boolean | null;
+    defaultProxyDisableError: string | null;
+    proxyResolution: KernelProxyResolutionDebug | null;
+    timeoutSeconds: number | null;
+    stealth: boolean | null;
+    profilePresent: boolean | null;
+    poolPresent: boolean | null;
+    retrieveError: string | null;
+}
+interface BrowserContextDebug {
+    viewport: {
+        width: number;
+        height: number;
+    };
+    locale: string;
+    device: SerpDevice;
+    userAgent: string;
+    deviceScaleFactor: number | null;
+    isMobile: boolean;
+    hasTouch: boolean;
+}
+interface SerpNavigationDebug {
+    requestedUrl: string;
+    finalUrl: string | null;
+    title: string | null;
+    bodySnippet: string | null;
+    hasPaa: boolean | null;
+    captchaDetected: boolean | null;
+    googleSorryUrl: boolean | null;
+    redirected: boolean | null;
+}
+interface BrowserDebugSnapshot {
+    kernel: KernelSessionDebug | null;
+    context: BrowserContextDebug | null;
+    networkLocation: BrowserNetworkLocationDebug | null;
+    serpNavigation: SerpNavigationDebug | null;
+}
+interface SerpLocationCandidate {
+    city: string;
+    regionCode: string;
+    count: number;
+    examples: string[];
+}
+interface SerpLocationEvidence {
+    status: 'not_requested' | 'matched' | 'mismatch' | 'unknown';
+    expected: {
+        city: string;
+        regionCode: string | null;
+        canonicalLocation: string;
+    } | null;
+    candidates: SerpLocationCandidate[];
+}
+interface HarvestDebugSnapshot {
+    enabled: true;
+    request: {
+        query: string;
+        locationInput: string | null;
+        canonicalLocation: string | null;
+        uule: string | null;
+        gl: string;
+        hl: string;
+        device: SerpDevice;
+        proxyMode: ProxyMode;
+        proxyZip: string | null;
+        serpOnly: boolean;
+        pages: number;
+    };
+    browser: BrowserDebugSnapshot;
+    locationEvidence?: SerpLocationEvidence;
+}
+interface PAANode {
+    question: string;
+    answer: string | null;
+    sourceTitle: string | null;
+    sourceSite: string | null;
+    sourceCite: string | null;
+    depth: number;
+    parentQuestion: string | null;
+    children: PAANode[];
+}
+interface FlatRow {
+    seed_query: string;
+    question: string;
+    answer: string;
+    source_title: string;
+    source_site: string;
+    source_cite: string;
+    depth: number;
+    parent_question: string;
+    extracted_at: string;
+}
+interface VideoResult {
+    type: 'video' | 'short_video';
+    title: string;
+    channel: string;
+    platform: string;
+    duration: string;
+    url: string;
+}
+interface ForumResult {
+    title: string;
+    source: string;
+    url: string;
+}
+interface WhatPeopleSayingCard {
+    type: 'reddit' | 'facebook' | 'instagram' | 'tiktok' | 'youtube' | 'news' | 'unknown';
+    title: string;
+    url: string;
+    source: string;
+    platform: string;
+    popularComment: string | null;
+    engagement: string;
+    date: string;
+    duration: string | null;
+    authorNote: string | null;
+}
+interface AICitation {
+    text: string;
+    href: string;
+}
+interface AIOverviewResult {
+    detected: boolean;
+    text: string | null;
+    citations: AICitation[];
+    expanded?: boolean;
+    fullyExpanded?: boolean;
+    sections?: string[];
+    shareUrl?: string | null;
+}
+interface AIModeResult {
+    detected: boolean;
+    text: string | null;
+    citations: AICitation[];
+}
+type GoogleSurface = 'web' | 'aim' | 'unknown';
+interface OrganicResult {
+    position: number;
+    title: string;
+    url: string;
+    domain: string;
+    cite: string | null;
+    snippet: string | null;
+    isRedditStyle: boolean;
+    inlineRating: {
+        value: string;
+        count: string;
+    } | null;
+}
+interface LocalPackBusiness {
+    position: number;
+    name: string;
+    cid: string | null;
+    rating: string | null;
+    reviewCount: string | null;
+    metadata: string[];
+    websiteUrl: string | null;
+    directionsUrl: string | null;
+}
+interface EntityRecord {
+    name: string;
+    kgId: string | null;
+    cid: string | null;
+    gcid: string | null;
+}
+interface EntityIds {
+    entities: EntityRecord[];
+    kgIds: string[];
+    cids: string[];
+    gcids: string[];
+}
+interface HarvestResult {
+    seed: string;
+    location: string | null;
+    extractedAt: string;
+    diagnostics: HarvestDiagnostics;
+    totalQuestions: number;
+    surface: GoogleSurface;
+    aiOverview: AIOverviewResult;
+    aiMode: AIModeResult;
+    whatPeopleSaying: WhatPeopleSayingCard[];
+    tree: PAANode[];
+    flat: FlatRow[];
+    videos: VideoResult[];
+    forums: ForumResult[];
+    organicResults: OrganicResult[];
+    localPack: LocalPackBusiness[];
+    entityIds: EntityIds;
+    stats: HarvestStats;
+}
+type HarvestOptions = z.infer<typeof HarvestOptionsSchema>;
+
+declare function harvest(rawOptions: unknown): Promise<HarvestResult>;
+
+interface EpisodeBrief {
+    episodeNumber: number;
+    episodeCount: number;
+    slug: string;
+    sectionTitle: string;
+    storyMoment: string;
+    characterName: string;
+    clip1: string;
+    clip2: string;
+    voiceover: string;
+    audioMood: string;
+    depositionVisualTell: string;
+    continuityNote: string;
+}
+
+interface ClipPairOptions {
+    resolution?: '480p' | '720p' | '1080p';
+    aspectRatio?: '16:9' | '9:16' | '1:1';
+    clipDurationSeconds?: number;
+    generateAudio?: boolean;
+    seed?: number;
+    outputDir?: string;
+}
+interface ClipPairResult {
+    clip1Url: string;
+    clip2Url: string;
+    finalVideoPath: string;
+    seed: number;
+    promptClip1: string;
+    promptClip2: string;
+    voiceover: string;
+    audioMood: string;
+}
+declare class VideoGenerator {
+    constructor(apiKey?: string);
+    generateClipPair(question: string, answer: string, opts?: ClipPairOptions): Promise<ClipPairResult>;
+    generateEpisode(brief: EpisodeBrief, opts?: ClipPairOptions): Promise<ClipPairResult>;
+}
+
+interface ClipPromptPair {
+    clip1: string;
+    clip2: string;
+    voiceover: string;
+    audioMood: string;
+}
+declare function buildClipPrompts(question: string, answer: string): Promise<ClipPromptPair>;
+
+export { type AICitation, type AIModeResult, type AIOverviewResult, type ClipPairOptions, type ClipPairResult, type FlatRow, type GoogleSurface, type HarvestCompletionStatus, type HarvestDiagnostics, type HarvestOptions, type HarvestResult, type HarvestStats, type PAANode, VideoGenerator, type WhatPeopleSayingCard, buildClipPrompts, harvest };