mcp-scraper 0.1.0 → 0.1.1

package/dist/index.d.cts

@@ -5,6 +5,10 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     location: z.ZodOptional<z.ZodString>;
     gl: z.ZodDefault<z.ZodString>;
     hl: z.ZodDefault<z.ZodString>;
+    device: z.ZodDefault<z.ZodEnum<["desktop", "mobile"]>>;
+    proxyMode: z.ZodDefault<z.ZodEnum<["location", "configured", "none"]>>;
+    proxyZip: z.ZodOptional<z.ZodString>;
+    debug: z.ZodDefault<z.ZodBoolean>;
     depth: z.ZodDefault<z.ZodNumber>;
     maxQuestions: z.ZodDefault<z.ZodNumber>;
     headless: z.ZodDefault<z.ZodBoolean>;
@@ -12,6 +16,7 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     proxy: z.ZodOptional<z.ZodString>;
     kernelApiKey: z.ZodOptional<z.ZodString>;
     kernelProxyId: z.ZodOptional<z.ZodString>;
+    kernelProxyResolution: z.ZodOptional<z.ZodUnknown>;
     outputDir: z.ZodDefault<z.ZodString>;
     format: z.ZodDefault<z.ZodEnum<["json", "csv", "both"]>>;
     serpOnly: z.ZodDefault<z.ZodBoolean>;
@@ -20,6 +25,9 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     query: string;
     gl: string;
     hl: string;
+    device: "desktop" | "mobile";
+    proxyMode: "location" | "configured" | "none";
+    debug: boolean;
     depth: number;
     maxQuestions: number;
     headless: boolean;
@@ -28,15 +36,21 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     serpOnly: boolean;
     pages: number;
     location?: string | undefined;
+    proxyZip?: string | undefined;
     profileDir?: string | undefined;
     proxy?: string | undefined;
     kernelApiKey?: string | undefined;
     kernelProxyId?: string | undefined;
+    kernelProxyResolution?: unknown;
 }, {
     query: string;
     location?: string | undefined;
     gl?: string | undefined;
     hl?: string | undefined;
+    device?: "desktop" | "mobile" | undefined;
+    proxyMode?: "location" | "configured" | "none" | undefined;
+    proxyZip?: string | undefined;
+    debug?: boolean | undefined;
     depth?: number | undefined;
     maxQuestions?: number | undefined;
     headless?: boolean | undefined;
@@ -44,12 +58,41 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     proxy?: string | undefined;
     kernelApiKey?: string | undefined;
     kernelProxyId?: string | undefined;
+    kernelProxyResolution?: unknown;
     outputDir?: string | undefined;
     format?: "json" | "csv" | "both" | undefined;
     serpOnly?: boolean | undefined;
     pages?: number | undefined;
 }>;
 
+type CanonicalLocationName = string;
+type SerpDevice = 'desktop' | 'mobile';
+type ProxyMode = 'location' | 'configured' | 'none';
+interface KernelLocationProxyTarget {
+    canonicalLocation: CanonicalLocationName;
+    level: 'zip' | 'city' | 'state';
+    country: 'US';
+    state: string;
+    city: string;
+    zip?: string;
+    cityCandidates: string[];
+    proxyName: string;
+    config: {
+        country: 'US';
+        state: string;
+        city?: string;
+        zip?: string;
+    };
+}
+type KernelProxyResolutionSource = 'disabled' | 'location_reused' | 'location_created' | 'configured_fallback' | 'unavailable';
+interface KernelProxyResolutionDebug {
+    source: KernelProxyResolutionSource;
+    proxyMode: ProxyMode;
+    proxyIdPresent: boolean;
+    proxyIdSuffix: string | null;
+    target: KernelLocationProxyTarget | null;
+    error: string | null;
+}
 interface HarvestStats {
     seed: string;
     totalQuestions: number;
@@ -57,6 +100,111 @@ interface HarvestStats {
     durationMs: number;
     errorCount: number;
 }
+type HarvestCompletionStatus = 'paa_found' | 'no_paa' | 'serp_only';
+type HarvestDiagnosticWarningCode = 'short_videos_captcha_skipped';
+type HarvestDiagnosticWarningSurface = 'short_videos';
+interface HarvestDiagnosticWarning {
+    code: HarvestDiagnosticWarningCode;
+    surface: HarvestDiagnosticWarningSurface;
+    message: string;
+    retryable: boolean;
+}
+interface HarvestDiagnostics {
+    completionStatus: HarvestCompletionStatus;
+    problem: null;
+    warnings?: HarvestDiagnosticWarning[];
+    debug?: HarvestDebugSnapshot;
+}
+interface BrowserNetworkLocationDebug {
+    source: 'ipapi.co' | 'ipwho.is' | 'api64.ipify.org';
+    ip: string | null;
+    city: string | null;
+    region: string | null;
+    country: string | null;
+    org: string | null;
+    timezone: string | null;
+    error: string | null;
+}
+interface KernelSessionDebug {
+    sessionId: string | null;
+    proxyMode: ProxyMode;
+    requestedProxyIdPresent: boolean;
+    requestedProxyIdSuffix: string | null;
+    createdProxyIdPresent: boolean | null;
+    createdProxyIdSuffix: string | null;
+    retrievedProxyIdPresent: boolean | null;
+    retrievedProxyIdSuffix: string | null;
+    retrievedProxyIdMatchesRequested: boolean | null;
+    defaultProxyDisabled: boolean | null;
+    defaultProxyDisableError: string | null;
+    proxyResolution: KernelProxyResolutionDebug | null;
+    timeoutSeconds: number | null;
+    stealth: boolean | null;
+    profilePresent: boolean | null;
+    poolPresent: boolean | null;
+    retrieveError: string | null;
+}
+interface BrowserContextDebug {
+    viewport: {
+        width: number;
+        height: number;
+    };
+    locale: string;
+    device: SerpDevice;
+    userAgent: string;
+    deviceScaleFactor: number | null;
+    isMobile: boolean;
+    hasTouch: boolean;
+}
+interface SerpNavigationDebug {
+    requestedUrl: string;
+    finalUrl: string | null;
+    title: string | null;
+    bodySnippet: string | null;
+    hasPaa: boolean | null;
+    captchaDetected: boolean | null;
+    googleSorryUrl: boolean | null;
+    redirected: boolean | null;
+}
+interface BrowserDebugSnapshot {
+    kernel: KernelSessionDebug | null;
+    context: BrowserContextDebug | null;
+    networkLocation: BrowserNetworkLocationDebug | null;
+    serpNavigation: SerpNavigationDebug | null;
+}
+interface SerpLocationCandidate {
+    city: string;
+    regionCode: string;
+    count: number;
+    examples: string[];
+}
+interface SerpLocationEvidence {
+    status: 'not_requested' | 'matched' | 'mismatch' | 'unknown';
+    expected: {
+        city: string;
+        regionCode: string | null;
+        canonicalLocation: string;
+    } | null;
+    candidates: SerpLocationCandidate[];
+}
+interface HarvestDebugSnapshot {
+    enabled: true;
+    request: {
+        query: string;
+        locationInput: string | null;
+        canonicalLocation: string | null;
+        uule: string | null;
+        gl: string;
+        hl: string;
+        device: SerpDevice;
+        proxyMode: ProxyMode;
+        proxyZip: string | null;
+        serpOnly: boolean;
+        pages: number;
+    };
+    browser: BrowserDebugSnapshot;
+    locationEvidence?: SerpLocationEvidence;
+}
 interface PAANode {
     question: string;
     answer: string | null;
@@ -111,6 +259,9 @@ interface AIOverviewResult {
     detected: boolean;
     text: string | null;
     citations: AICitation[];
+    expanded?: boolean;
+    fullyExpanded?: boolean;
+    sections?: string[];
 }
 interface AIModeResult {
     detected: boolean;
@@ -157,6 +308,7 @@ interface HarvestResult {
     seed: string;
     location: string | null;
     extractedAt: string;
+    diagnostics: HarvestDiagnostics;
     totalQuestions: number;
     surface: GoogleSurface;
     aiOverview: AIOverviewResult;
@@ -175,6 +327,21 @@ type HarvestOptions = z.infer<typeof HarvestOptionsSchema>;
 
 declare function harvest(rawOptions: unknown): Promise<HarvestResult>;
 
+interface EpisodeBrief {
+    episodeNumber: number;
+    episodeCount: number;
+    slug: string;
+    sectionTitle: string;
+    storyMoment: string;
+    characterName: string;
+    clip1: string;
+    clip2: string;
+    voiceover: string;
+    audioMood: string;
+    depositionVisualTell: string;
+    continuityNote: string;
+}
+
 interface ClipPairOptions {
     resolution?: '480p' | '720p' | '1080p';
     aspectRatio?: '16:9' | '9:16' | '1:1';
@@ -182,7 +349,6 @@ interface ClipPairOptions {
     generateAudio?: boolean;
     seed?: number;
     outputDir?: string;
-    ttsVoice?: string;
 }
 interface ClipPairResult {
     clip1Url: string;
@@ -197,6 +363,7 @@ interface ClipPairResult {
 declare class VideoGenerator {
     constructor(apiKey?: string);
     generateClipPair(question: string, answer: string, opts?: ClipPairOptions): Promise<ClipPairResult>;
+    generateEpisode(brief: EpisodeBrief, opts?: ClipPairOptions): Promise<ClipPairResult>;
 }
 
 interface ClipPromptPair {
@@ -207,4 +374,4 @@ interface ClipPromptPair {
 }
 declare function buildClipPrompts(question: string, answer: string): Promise<ClipPromptPair>;
 
-export { type AICitation, type AIModeResult, type AIOverviewResult, type ClipPairOptions, type ClipPairResult, type FlatRow, type GoogleSurface, type HarvestOptions, type HarvestResult, type HarvestStats, type PAANode, VideoGenerator, type WhatPeopleSayingCard, buildClipPrompts, harvest };
+export { type AICitation, type AIModeResult, type AIOverviewResult, type ClipPairOptions, type ClipPairResult, type FlatRow, type GoogleSurface, type HarvestCompletionStatus, type HarvestDiagnostics, type HarvestOptions, type HarvestResult, type HarvestStats, type PAANode, VideoGenerator, type WhatPeopleSayingCard, buildClipPrompts, harvest };

package/dist/index.d.ts

@@ -5,6 +5,10 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     location: z.ZodOptional<z.ZodString>;
     gl: z.ZodDefault<z.ZodString>;
     hl: z.ZodDefault<z.ZodString>;
+    device: z.ZodDefault<z.ZodEnum<["desktop", "mobile"]>>;
+    proxyMode: z.ZodDefault<z.ZodEnum<["location", "configured", "none"]>>;
+    proxyZip: z.ZodOptional<z.ZodString>;
+    debug: z.ZodDefault<z.ZodBoolean>;
     depth: z.ZodDefault<z.ZodNumber>;
     maxQuestions: z.ZodDefault<z.ZodNumber>;
     headless: z.ZodDefault<z.ZodBoolean>;
@@ -12,6 +16,7 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     proxy: z.ZodOptional<z.ZodString>;
     kernelApiKey: z.ZodOptional<z.ZodString>;
     kernelProxyId: z.ZodOptional<z.ZodString>;
+    kernelProxyResolution: z.ZodOptional<z.ZodUnknown>;
     outputDir: z.ZodDefault<z.ZodString>;
     format: z.ZodDefault<z.ZodEnum<["json", "csv", "both"]>>;
     serpOnly: z.ZodDefault<z.ZodBoolean>;
@@ -20,6 +25,9 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     query: string;
     gl: string;
     hl: string;
+    device: "desktop" | "mobile";
+    proxyMode: "location" | "configured" | "none";
+    debug: boolean;
     depth: number;
     maxQuestions: number;
     headless: boolean;
@@ -28,15 +36,21 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     serpOnly: boolean;
     pages: number;
     location?: string | undefined;
+    proxyZip?: string | undefined;
     profileDir?: string | undefined;
     proxy?: string | undefined;
     kernelApiKey?: string | undefined;
     kernelProxyId?: string | undefined;
+    kernelProxyResolution?: unknown;
 }, {
     query: string;
     location?: string | undefined;
     gl?: string | undefined;
     hl?: string | undefined;
+    device?: "desktop" | "mobile" | undefined;
+    proxyMode?: "location" | "configured" | "none" | undefined;
+    proxyZip?: string | undefined;
+    debug?: boolean | undefined;
     depth?: number | undefined;
     maxQuestions?: number | undefined;
     headless?: boolean | undefined;
@@ -44,12 +58,41 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     proxy?: string | undefined;
     kernelApiKey?: string | undefined;
     kernelProxyId?: string | undefined;
+    kernelProxyResolution?: unknown;
     outputDir?: string | undefined;
     format?: "json" | "csv" | "both" | undefined;
     serpOnly?: boolean | undefined;
     pages?: number | undefined;
 }>;
 
+type CanonicalLocationName = string;
+type SerpDevice = 'desktop' | 'mobile';
+type ProxyMode = 'location' | 'configured' | 'none';
+interface KernelLocationProxyTarget {
+    canonicalLocation: CanonicalLocationName;
+    level: 'zip' | 'city' | 'state';
+    country: 'US';
+    state: string;
+    city: string;
+    zip?: string;
+    cityCandidates: string[];
+    proxyName: string;
+    config: {
+        country: 'US';
+        state: string;
+        city?: string;
+        zip?: string;
+    };
+}
+type KernelProxyResolutionSource = 'disabled' | 'location_reused' | 'location_created' | 'configured_fallback' | 'unavailable';
+interface KernelProxyResolutionDebug {
+    source: KernelProxyResolutionSource;
+    proxyMode: ProxyMode;
+    proxyIdPresent: boolean;
+    proxyIdSuffix: string | null;
+    target: KernelLocationProxyTarget | null;
+    error: string | null;
+}
 interface HarvestStats {
     seed: string;
     totalQuestions: number;
@@ -57,6 +100,111 @@ interface HarvestStats {
     durationMs: number;
     errorCount: number;
 }
+type HarvestCompletionStatus = 'paa_found' | 'no_paa' | 'serp_only';
+type HarvestDiagnosticWarningCode = 'short_videos_captcha_skipped';
+type HarvestDiagnosticWarningSurface = 'short_videos';
+interface HarvestDiagnosticWarning {
+    code: HarvestDiagnosticWarningCode;
+    surface: HarvestDiagnosticWarningSurface;
+    message: string;
+    retryable: boolean;
+}
+interface HarvestDiagnostics {
+    completionStatus: HarvestCompletionStatus;
+    problem: null;
+    warnings?: HarvestDiagnosticWarning[];
+    debug?: HarvestDebugSnapshot;
+}
+interface BrowserNetworkLocationDebug {
+    source: 'ipapi.co' | 'ipwho.is' | 'api64.ipify.org';
+    ip: string | null;
+    city: string | null;
+    region: string | null;
+    country: string | null;
+    org: string | null;
+    timezone: string | null;
+    error: string | null;
+}
+interface KernelSessionDebug {
+    sessionId: string | null;
+    proxyMode: ProxyMode;
+    requestedProxyIdPresent: boolean;
+    requestedProxyIdSuffix: string | null;
+    createdProxyIdPresent: boolean | null;
+    createdProxyIdSuffix: string | null;
+    retrievedProxyIdPresent: boolean | null;
+    retrievedProxyIdSuffix: string | null;
+    retrievedProxyIdMatchesRequested: boolean | null;
+    defaultProxyDisabled: boolean | null;
+    defaultProxyDisableError: string | null;
+    proxyResolution: KernelProxyResolutionDebug | null;
+    timeoutSeconds: number | null;
+    stealth: boolean | null;
+    profilePresent: boolean | null;
+    poolPresent: boolean | null;
+    retrieveError: string | null;
+}
+interface BrowserContextDebug {
+    viewport: {
+        width: number;
+        height: number;
+    };
+    locale: string;
+    device: SerpDevice;
+    userAgent: string;
+    deviceScaleFactor: number | null;
+    isMobile: boolean;
+    hasTouch: boolean;
+}
+interface SerpNavigationDebug {
+    requestedUrl: string;
+    finalUrl: string | null;
+    title: string | null;
+    bodySnippet: string | null;
+    hasPaa: boolean | null;
+    captchaDetected: boolean | null;
+    googleSorryUrl: boolean | null;
+    redirected: boolean | null;
+}
+interface BrowserDebugSnapshot {
+    kernel: KernelSessionDebug | null;
+    context: BrowserContextDebug | null;
+    networkLocation: BrowserNetworkLocationDebug | null;
+    serpNavigation: SerpNavigationDebug | null;
+}
+interface SerpLocationCandidate {
+    city: string;
+    regionCode: string;
+    count: number;
+    examples: string[];
+}
+interface SerpLocationEvidence {
+    status: 'not_requested' | 'matched' | 'mismatch' | 'unknown';
+    expected: {
+        city: string;
+        regionCode: string | null;
+        canonicalLocation: string;
+    } | null;
+    candidates: SerpLocationCandidate[];
+}
+interface HarvestDebugSnapshot {
+    enabled: true;
+    request: {
+        query: string;
+        locationInput: string | null;
+        canonicalLocation: string | null;
+        uule: string | null;
+        gl: string;
+        hl: string;
+        device: SerpDevice;
+        proxyMode: ProxyMode;
+        proxyZip: string | null;
+        serpOnly: boolean;
+        pages: number;
+    };
+    browser: BrowserDebugSnapshot;
+    locationEvidence?: SerpLocationEvidence;
+}
 interface PAANode {
     question: string;
     answer: string | null;
@@ -111,6 +259,9 @@ interface AIOverviewResult {
     detected: boolean;
     text: string | null;
     citations: AICitation[];
+    expanded?: boolean;
+    fullyExpanded?: boolean;
+    sections?: string[];
 }
 interface AIModeResult {
     detected: boolean;
@@ -157,6 +308,7 @@ interface HarvestResult {
     seed: string;
     location: string | null;
     extractedAt: string;
+    diagnostics: HarvestDiagnostics;
     totalQuestions: number;
     surface: GoogleSurface;
     aiOverview: AIOverviewResult;
@@ -175,6 +327,21 @@ type HarvestOptions = z.infer<typeof HarvestOptionsSchema>;
 
 declare function harvest(rawOptions: unknown): Promise<HarvestResult>;
 
+interface EpisodeBrief {
+    episodeNumber: number;
+    episodeCount: number;
+    slug: string;
+    sectionTitle: string;
+    storyMoment: string;
+    characterName: string;
+    clip1: string;
+    clip2: string;
+    voiceover: string;
+    audioMood: string;
+    depositionVisualTell: string;
+    continuityNote: string;
+}
+
 interface ClipPairOptions {
     resolution?: '480p' | '720p' | '1080p';
     aspectRatio?: '16:9' | '9:16' | '1:1';
@@ -182,7 +349,6 @@ interface ClipPairOptions {
     generateAudio?: boolean;
     seed?: number;
     outputDir?: string;
-    ttsVoice?: string;
 }
 interface ClipPairResult {
     clip1Url: string;
@@ -197,6 +363,7 @@ interface ClipPairResult {
 declare class VideoGenerator {
     constructor(apiKey?: string);
     generateClipPair(question: string, answer: string, opts?: ClipPairOptions): Promise<ClipPairResult>;
+    generateEpisode(brief: EpisodeBrief, opts?: ClipPairOptions): Promise<ClipPairResult>;
 }
 
 interface ClipPromptPair {
@@ -207,4 +374,4 @@ interface ClipPromptPair {
 }
 declare function buildClipPrompts(question: string, answer: string): Promise<ClipPromptPair>;
 
-export { type AICitation, type AIModeResult, type AIOverviewResult, type ClipPairOptions, type ClipPairResult, type FlatRow, type GoogleSurface, type HarvestOptions, type HarvestResult, type HarvestStats, type PAANode, VideoGenerator, type WhatPeopleSayingCard, buildClipPrompts, harvest };
+export { type AICitation, type AIModeResult, type AIOverviewResult, type ClipPairOptions, type ClipPairResult, type FlatRow, type GoogleSurface, type HarvestCompletionStatus, type HarvestDiagnostics, type HarvestOptions, type HarvestResult, type HarvestStats, type PAANode, VideoGenerator, type WhatPeopleSayingCard, buildClipPrompts, harvest };