mcp-scraper 0.92.1 → 0.93.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/CHANGELOG.md +45 -26
  2. package/README.md +4 -4
  3. package/THIRD_PARTY_NOTICES.html +203 -0
  4. package/dist/bin/api-server.js +1 -1
  5. package/dist/bin/mcp-scraper-cli.js +1 -1
  6. package/dist/bin/mcp-scraper-core.js +1 -1
  7. package/dist/bin/mcp-scraper-install.js +1 -1
  8. package/dist/bin/mcp-stdio-server.js +1 -1
  9. package/dist/bin/paa-harvest.js +1 -1
  10. package/dist/chunk-2SP57VCG.js +15 -15
  11. package/dist/{chunk-QHHOKLQY.js → chunk-5JKBFNYF.js} +1 -1
  12. package/dist/{chunk-ZJZMDWDI.js → chunk-5QYW4CSJ.js} +5 -5
  13. package/dist/{chunk-YXUPFECS.js → chunk-AFFMCO7R.js} +2 -2
  14. package/dist/chunk-AYQSKD7D.js +1 -0
  15. package/dist/chunk-DT2FYN6N.js +1 -0
  16. package/dist/{chunk-TCLSPE3P.js → chunk-IJF4VXJ7.js} +1 -1
  17. package/dist/chunk-KPQM4UDJ.js +6 -0
  18. package/dist/chunk-M5QHXNFZ.js +3 -3
  19. package/dist/{chunk-ZMNUI5LB.js → chunk-M5VZVMPC.js} +8 -8
  20. package/dist/{chunk-T2IJVEHG.js → chunk-NLKA6SHC.js} +1 -1
  21. package/dist/{chunk-QXAXPIHM.js → chunk-OD2WLHEN.js} +243 -240
  22. package/dist/chunk-ORB4RHCK.js +4 -4
  23. package/dist/chunk-SH5KB4P7.js +1 -0
  24. package/dist/chunk-UN6FVDJQ.js +1 -0
  25. package/dist/{chunk-JHMI6HEO.js → chunk-W2T4NTCE.js} +1 -1
  26. package/dist/chunk-WJ4XFLS4.js +15 -15
  27. package/dist/{chunk-GPZ7ITF6.js → chunk-X6RD3Y4W.js} +1 -1
  28. package/dist/chunk-Y6MKMSOC.js +1 -0
  29. package/dist/{extract-bundle-5QTPVIWO.js → extract-bundle-C3N6E6V7.js} +1 -1
  30. package/dist/{gmail-service-EOC7KJSK.js → gmail-service-6V5MFBYH.js} +1 -1
  31. package/dist/index.cjs +57 -57
  32. package/dist/index.d.cts +162 -121
  33. package/dist/index.d.ts +162 -121
  34. package/dist/index.js +6 -6
  35. package/dist/{server-NAQTBNPX.js → server-KWGXYWYO.js} +71 -71
  36. package/dist/{site-extract-repository-U4G25TR2.js → site-extract-repository-2LFDP6FZ.js} +1 -1
  37. package/dist/{stripe-event-worker-KW3DR6GB.js → stripe-event-worker-IJXA6T37.js} +1 -1
  38. package/dist/worker-ASXALAY7.js +1 -0
  39. package/package.json +17 -136
  40. package/dist/chunk-2BN2TB3U.js +0 -1
  41. package/dist/chunk-2TZBO52D.js +0 -1
  42. package/dist/chunk-4FROKQJN.js +0 -1
  43. package/dist/chunk-GCLSGYYE.js +0 -1
  44. package/dist/chunk-K6MN7UED.js +0 -2
  45. package/dist/chunk-RECKUOEF.js +0 -1
  46. package/dist/chunk-RSPA4IWH.js +0 -1
  47. package/dist/worker-MSLXSR4M.js +0 -1
package/dist/index.d.cts CHANGED
@@ -1,5 +1,44 @@
1
1
  import { z } from 'zod';
2
2
 
3
+ declare const SERP_FEATURE_COMPLETENESS_STATES: readonly ["observed_present", "observed_absent", "incomplete", "not_requested", "unknown", "unsupported"];
4
+ type SerpFeatureCompletenessState = typeof SERP_FEATURE_COMPLETENESS_STATES[number];
5
+
6
+ interface SerpPreviewQuestion {
7
+ question: string;
8
+ answer: string | null;
9
+ url: string | null;
10
+ }
11
+ interface SerpFeatureItem {
12
+ title: string | null;
13
+ url: string | null;
14
+ description: string | null;
15
+ position: number | null;
16
+ }
17
+ interface SerpFeatureGroup {
18
+ name: string;
19
+ attributes: Record<string, string | number | boolean | null>;
20
+ items: SerpFeatureItem[];
21
+ }
22
+ interface FullSerpPageFeatures {
23
+ page: 1 | 2;
24
+ status: Record<string, SerpFeatureCompletenessState>;
25
+ paaPreview: SerpPreviewQuestion[];
26
+ additional: SerpFeatureGroup[];
27
+ localPack: LocalPackBusiness[];
28
+ forums: ForumResult[];
29
+ videos: VideoResult[];
30
+ whatPeopleSaying: WhatPeopleSayingCard[];
31
+ aiOverview: AIOverviewResult;
32
+ aiMode: AIModeResult;
33
+ entityIds: EntityIds;
34
+ }
35
+ interface FullSerpFeatures {
36
+ status: Record<string, SerpFeatureCompletenessState>;
37
+ paaPreview: SerpPreviewQuestion[];
38
+ additional: SerpFeatureGroup[];
39
+ pages: FullSerpPageFeatures[];
40
+ }
41
+
3
42
  declare const HarvestOptionsSchema: z.ZodObject<{
4
43
  query: z.ZodString;
5
44
  location: z.ZodOptional<z.ZodString>;
@@ -24,13 +63,13 @@ declare const HarvestOptionsSchema: z.ZodObject<{
24
63
  headless: z.ZodDefault<z.ZodBoolean>;
25
64
  profileDir: z.ZodOptional<z.ZodString>;
26
65
  proxy: z.ZodOptional<z.ZodString>;
27
- kernelApiKey: z.ZodOptional<z.ZodString>;
28
- kernelProxyId: z.ZodOptional<z.ZodString>;
29
- kernelProfileName: z.ZodOptional<z.ZodString>;
30
- kernelProfileSaveChanges: z.ZodOptional<z.ZodBoolean>;
31
- kernelStealth: z.ZodOptional<z.ZodBoolean>;
66
+ \u006b\u0065\u0072\u006e\u0065\u006cApiKey: z.ZodOptional<z.ZodString>;
67
+ \u006b\u0065\u0072\u006e\u0065\u006cProxyId: z.ZodOptional<z.ZodString>;
68
+ \u006b\u0065\u0072\u006e\u0065\u006cProfileName: z.ZodOptional<z.ZodString>;
69
+ \u006b\u0065\u0072\u006e\u0065\u006cProfileSaveChanges: z.ZodOptional<z.ZodBoolean>;
70
+ \u006b\u0065\u0072\u006e\u0065\u006cStealth: z.ZodOptional<z.ZodBoolean>;
32
71
  serpIdentity: z.ZodOptional<z.ZodString>;
33
- kernelProxyResolution: z.ZodOptional<z.ZodUnknown>;
72
+ \u006b\u0065\u0072\u006e\u0065\u006cProxyResolution: z.ZodOptional<z.ZodUnknown>;
34
73
  outputDir: z.ZodDefault<z.ZodString>;
35
74
  format: z.ZodDefault<z.ZodEnum<{
36
75
  json: "json";
@@ -57,75 +96,6 @@ declare const HarvestOptionsSchema: z.ZodObject<{
57
96
  softDeadlineMs: z.ZodOptional<z.ZodNumber>;
58
97
  }, z.core.$strip>;
59
98
 
60
- type CanonicalLocationName = string;
61
- type SerpDevice = 'desktop' | 'mobile';
62
- type ProxyMode = 'location' | 'configured' | 'none';
63
- interface KernelLocationProxyTarget {
64
- canonicalLocation: CanonicalLocationName;
65
- level: 'zip' | 'city' | 'state';
66
- country: 'US';
67
- state: string;
68
- city: string;
69
- zip?: string;
70
- cityCandidates: string[];
71
- proxyName: string;
72
- config: {
73
- country: 'US';
74
- state: string;
75
- city?: string;
76
- zip?: string;
77
- };
78
- }
79
- type KernelProxyResolutionSource = 'disabled' | 'location_reused' | 'location_created' | 'isp_created' | 'backup_custom' | 'configured_fallback' | 'unavailable';
80
- interface KernelProxyResolutionDebug {
81
- source: KernelProxyResolutionSource;
82
- proxyType: string;
83
- proxyMode: ProxyMode;
84
- proxyIdPresent: boolean;
85
- proxyIdSuffix: string | null;
86
- target: KernelLocationProxyTarget | null;
87
- error: string | null;
88
- disposable?: boolean;
89
- }
90
- interface HarvestStats {
91
- seed: string;
92
- totalQuestions: number;
93
- maxDepthReached: number;
94
- durationMs: number;
95
- errorCount: number;
96
- }
97
- type HarvestCompletionStatus = 'paa_found' | 'no_paa' | 'serp_only';
98
- /**
99
- * A completed browser attempt can still be unsuitable for programmatic use.
100
- * This is deliberately separate from completionStatus so a valid no-PAA
101
- * response with organic results is not confused with an unreadable SERP.
102
- */
103
- type HarvestResultQuality = 'complete' | 'partial' | 'degraded';
104
- type HarvestDegradationReason = 'empty_primary_serp';
105
- type HarvestDiagnosticWarningCode = 'short_videos_captcha_skipped' | 'paa_partial';
106
- type HarvestDiagnosticWarningSurface = 'short_videos' | 'paa';
107
- interface HarvestDiagnosticWarning {
108
- code: HarvestDiagnosticWarningCode;
109
- surface: HarvestDiagnosticWarningSurface;
110
- message: string;
111
- retryable: boolean;
112
- }
113
- interface HarvestCompleteness {
114
- paaWithoutAnswer: number;
115
- paaWithoutSource: number;
116
- paaAnswersRecovered: number;
117
- aioShareCaptured: boolean | null;
118
- paaRequested?: number;
119
- paaReturned?: number;
120
- paaUnique?: number;
121
- }
122
- type HarvestInterruptionCode = 'browser_session_interrupted' | 'request_aborted' | 'timeout';
123
- type HarvestInterruptionReasonSource = 'client_runtime' | 'extractor';
124
- interface HarvestInterruptionDiagnostic {
125
- code: HarvestInterruptionCode;
126
- reasonSource: HarvestInterruptionReasonSource;
127
- message: string | null;
128
- }
129
99
  type PaaDiscoveryStatus = 'in_progress' | 'target_reached' | 'frontier_exhausted' | 'no_paa_observed' | 'interrupted' | 'recovery_exhausted';
130
100
  type PaaTargetStatus = 'met' | 'not_met';
131
101
  type PaaMaterialStatus = 'complete' | 'partial';
@@ -185,15 +155,6 @@ interface PaaInteractionObservation {
185
155
  errorCode: 'click_failed' | 'click_ack_timeout' | 'control_missing' | 'confirmation_timeout' | null;
186
156
  }
187
157
  type PaaProgressPhase = 'serp_captured' | 'discovered' | 'click_observed' | 'answer_captured' | 'fill_incomplete' | 'expansion_finished' | 'attempt_ending';
188
- /** Observed organic pagination outcome; page two never expands a separate PAA graph. */
189
- interface HarvestPaginationDiagnostics {
190
- requestedPages: 1 | 2;
191
- capturedPages: 1 | 2;
192
- page2Status: 'not_requested' | 'not_attempted' | 'captured' | 'unavailable' | 'failed';
193
- page1OrganicCount: number;
194
- page2OrganicCount: number;
195
- failureCode?: 'missing_next' | 'invalid_next' | 'empty_page' | 'captcha' | 'timeout' | 'navigation_error' | 'unsupported_driver';
196
- }
197
158
  /** Material already observed before PAA expansion completes. It is checkpointed
198
159
  * so a worker interruption cannot discard the SERP that led to the PAA graph. */
199
160
  interface PaaProgressMaterial {
@@ -218,6 +179,119 @@ interface PaaProgressSnapshot {
218
179
  material?: PaaProgressMaterial;
219
180
  interactions?: PaaInteractionObservation[];
220
181
  }
182
+ interface PaaSource extends SerpOutboundLink {
183
+ title: string | null;
184
+ site: string | null;
185
+ }
186
+ interface PAANode {
187
+ questionId: string;
188
+ googleLinkId: string | null;
189
+ googleEvidenceId: string | null;
190
+ question: string;
191
+ answer: string | null;
192
+ sourceTitle: string | null;
193
+ sourceSite: string | null;
194
+ sourceCite: string | null;
195
+ sources: PaaSource[];
196
+ depth: number;
197
+ parentQuestion: string | null;
198
+ children: PAANode[];
199
+ }
200
+ interface FlatRow {
201
+ seed_query: string;
202
+ question_id: string;
203
+ google_link_id: string;
204
+ google_evidence_id: string;
205
+ question: string;
206
+ answer: string;
207
+ source_title: string;
208
+ source_site: string;
209
+ source_cite: string;
210
+ sources: PaaSource[];
211
+ depth: number;
212
+ parent_question: string;
213
+ extracted_at: string;
214
+ }
215
+ type HarvestOptions = z.infer<typeof HarvestOptionsSchema>;
216
+
217
+ type CanonicalLocationName = string;
218
+ type SerpDevice = 'desktop' | 'mobile';
219
+ type ProxyMode = 'location' | 'configured' | 'none';
220
+ interface \u004b\u0065\u0072\u006e\u0065\u006cLocationProxyTarget {
221
+ canonicalLocation: CanonicalLocationName;
222
+ level: 'zip' | 'city' | 'state';
223
+ country: 'US';
224
+ state: string;
225
+ city: string;
226
+ zip?: string;
227
+ cityCandidates: string[];
228
+ proxyName: string;
229
+ config: {
230
+ country: 'US';
231
+ state: string;
232
+ city?: string;
233
+ zip?: string;
234
+ };
235
+ }
236
+ type \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionSource = 'disabled' | 'location_reused' | 'location_created' | 'isp_created' | 'backup_custom' | 'configured_fallback' | 'unavailable';
237
+ interface \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionDebug {
238
+ source: \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionSource;
239
+ proxyType: string;
240
+ proxyMode: ProxyMode;
241
+ proxyIdPresent: boolean;
242
+ proxyIdSuffix: string | null;
243
+ target: \u004b\u0065\u0072\u006e\u0065\u006cLocationProxyTarget | null;
244
+ error: string | null;
245
+ disposable?: boolean;
246
+ }
247
+ interface HarvestStats {
248
+ seed: string;
249
+ totalQuestions: number;
250
+ maxDepthReached: number;
251
+ durationMs: number;
252
+ errorCount: number;
253
+ }
254
+ type HarvestCompletionStatus = 'paa_found' | 'no_paa' | 'serp_only';
255
+ /**
256
+ * A completed browser attempt can still be unsuitable for programmatic use.
257
+ * This is deliberately separate from completionStatus so a valid no-PAA
258
+ * response with organic results is not confused with an unreadable SERP.
259
+ */
260
+ type HarvestResultQuality = 'complete' | 'partial' | 'degraded';
261
+ type HarvestDegradationReason = 'empty_primary_serp' | 'page_two_missing';
262
+ type HarvestDiagnosticWarningCode = 'short_videos_captcha_skipped' | 'paa_partial';
263
+ type HarvestDiagnosticWarningSurface = 'short_videos' | 'paa';
264
+ interface HarvestDiagnosticWarning {
265
+ code: HarvestDiagnosticWarningCode;
266
+ surface: HarvestDiagnosticWarningSurface;
267
+ message: string;
268
+ retryable: boolean;
269
+ }
270
+ interface HarvestCompleteness {
271
+ paaWithoutAnswer: number;
272
+ paaWithoutSource: number;
273
+ paaAnswersRecovered: number;
274
+ aioShareCaptured: boolean | null;
275
+ paaRequested?: number;
276
+ paaReturned?: number;
277
+ paaUnique?: number;
278
+ }
279
+ type HarvestInterruptionCode = 'browser_session_interrupted' | 'request_aborted' | 'timeout';
280
+ type HarvestInterruptionReasonSource = 'client_runtime' | 'extractor';
281
+ interface HarvestInterruptionDiagnostic {
282
+ code: HarvestInterruptionCode;
283
+ reasonSource: HarvestInterruptionReasonSource;
284
+ message: string | null;
285
+ }
286
+ /** Observed organic pagination outcome; page two never expands a separate PAA graph. */
287
+ interface HarvestPaginationDiagnostics {
288
+ requestedPages: 1 | 2;
289
+ capturedPages: 1 | 2;
290
+ page2Status: 'not_requested' | 'not_attempted' | 'captured' | 'unavailable' | 'failed';
291
+ page1OrganicCount: number;
292
+ page2OrganicCount: number;
293
+ failureCode?: 'missing_next' | 'invalid_next' | 'empty_page' | 'captcha' | 'timeout' | 'navigation_error' | 'unsupported_driver';
294
+ }
221
295
  interface HarvestDiagnostics {
222
296
  phaseTimings?: {
223
297
  providerConnectMs?: number;
@@ -265,7 +339,7 @@ interface BrowserNetworkLocationDebug {
265
339
  timezone: string | null;
266
340
  error: string | null;
267
341
  }
268
- interface KernelSessionDebug {
342
+ interface \u004b\u0065\u0072\u006e\u0065\u006cSessionDebug {
269
343
  sessionId: string | null;
270
344
  proxyMode: ProxyMode;
271
345
  requestedProxyIdPresent: boolean;
@@ -277,7 +351,7 @@ interface KernelSessionDebug {
277
351
  retrievedProxyIdMatchesRequested: boolean | null;
278
352
  defaultProxyDisabled: boolean | null;
279
353
  defaultProxyDisableError: string | null;
280
- proxyResolution: KernelProxyResolutionDebug | null;
354
+ proxyResolution: \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionDebug | null;
281
355
  timeoutSeconds: number | null;
282
356
  stealth: boolean | null;
283
357
  profilePresent: boolean | null;
@@ -307,7 +381,7 @@ interface SerpNavigationDebug {
307
381
  redirected: boolean | null;
308
382
  }
309
383
  interface BrowserDebugSnapshot {
310
- kernel: KernelSessionDebug | null;
384
+ \u006b\u0065\u0072\u006e\u0065\u006c: \u004b\u0065\u0072\u006e\u0065\u006cSessionDebug | null;
311
385
  context: BrowserContextDebug | null;
312
386
  networkLocation: BrowserNetworkLocationDebug | null;
313
387
  serpNavigation: SerpNavigationDebug | null;
@@ -358,39 +432,6 @@ interface SerpOutboundLink {
358
432
  linkType: SerpLinkType;
359
433
  resolutionStatus: SerpLinkResolutionStatus;
360
434
  }
361
- interface PaaSource extends SerpOutboundLink {
362
- title: string | null;
363
- site: string | null;
364
- }
365
- interface PAANode {
366
- questionId: string;
367
- googleLinkId: string | null;
368
- googleEvidenceId: string | null;
369
- question: string;
370
- answer: string | null;
371
- sourceTitle: string | null;
372
- sourceSite: string | null;
373
- sourceCite: string | null;
374
- sources: PaaSource[];
375
- depth: number;
376
- parentQuestion: string | null;
377
- children: PAANode[];
378
- }
379
- interface FlatRow {
380
- seed_query: string;
381
- question_id: string;
382
- google_link_id: string;
383
- google_evidence_id: string;
384
- question: string;
385
- answer: string;
386
- source_title: string;
387
- source_site: string;
388
- source_cite: string;
389
- sources: PaaSource[];
390
- depth: number;
391
- parent_question: string;
392
- extracted_at: string;
393
- }
394
435
  interface VideoResult {
395
436
  type: 'video' | 'short_video';
396
437
  title: string;
@@ -495,9 +536,9 @@ interface HarvestResult {
495
536
  organicResults: OrganicResult[];
496
537
  localPack: LocalPackBusiness[];
497
538
  entityIds: EntityIds;
539
+ fullSerpFeatures?: FullSerpFeatures;
498
540
  stats: HarvestStats;
499
541
  }
500
- type HarvestOptions = z.infer<typeof HarvestOptionsSchema>;
501
542
 
502
543
  declare function harvest(rawOptions: unknown): Promise<HarvestResult>;
503
544
 
package/dist/index.d.ts CHANGED
@@ -1,5 +1,44 @@
1
1
  import { z } from 'zod';
2
2
 
3
+ declare const SERP_FEATURE_COMPLETENESS_STATES: readonly ["observed_present", "observed_absent", "incomplete", "not_requested", "unknown", "unsupported"];
4
+ type SerpFeatureCompletenessState = typeof SERP_FEATURE_COMPLETENESS_STATES[number];
5
+
6
+ interface SerpPreviewQuestion {
7
+ question: string;
8
+ answer: string | null;
9
+ url: string | null;
10
+ }
11
+ interface SerpFeatureItem {
12
+ title: string | null;
13
+ url: string | null;
14
+ description: string | null;
15
+ position: number | null;
16
+ }
17
+ interface SerpFeatureGroup {
18
+ name: string;
19
+ attributes: Record<string, string | number | boolean | null>;
20
+ items: SerpFeatureItem[];
21
+ }
22
+ interface FullSerpPageFeatures {
23
+ page: 1 | 2;
24
+ status: Record<string, SerpFeatureCompletenessState>;
25
+ paaPreview: SerpPreviewQuestion[];
26
+ additional: SerpFeatureGroup[];
27
+ localPack: LocalPackBusiness[];
28
+ forums: ForumResult[];
29
+ videos: VideoResult[];
30
+ whatPeopleSaying: WhatPeopleSayingCard[];
31
+ aiOverview: AIOverviewResult;
32
+ aiMode: AIModeResult;
33
+ entityIds: EntityIds;
34
+ }
35
+ interface FullSerpFeatures {
36
+ status: Record<string, SerpFeatureCompletenessState>;
37
+ paaPreview: SerpPreviewQuestion[];
38
+ additional: SerpFeatureGroup[];
39
+ pages: FullSerpPageFeatures[];
40
+ }
41
+
3
42
  declare const HarvestOptionsSchema: z.ZodObject<{
4
43
  query: z.ZodString;
5
44
  location: z.ZodOptional<z.ZodString>;
@@ -24,13 +63,13 @@ declare const HarvestOptionsSchema: z.ZodObject<{
24
63
  headless: z.ZodDefault<z.ZodBoolean>;
25
64
  profileDir: z.ZodOptional<z.ZodString>;
26
65
  proxy: z.ZodOptional<z.ZodString>;
27
- kernelApiKey: z.ZodOptional<z.ZodString>;
28
- kernelProxyId: z.ZodOptional<z.ZodString>;
29
- kernelProfileName: z.ZodOptional<z.ZodString>;
30
- kernelProfileSaveChanges: z.ZodOptional<z.ZodBoolean>;
31
- kernelStealth: z.ZodOptional<z.ZodBoolean>;
66
+ \u006b\u0065\u0072\u006e\u0065\u006cApiKey: z.ZodOptional<z.ZodString>;
67
+ \u006b\u0065\u0072\u006e\u0065\u006cProxyId: z.ZodOptional<z.ZodString>;
68
+ \u006b\u0065\u0072\u006e\u0065\u006cProfileName: z.ZodOptional<z.ZodString>;
69
+ \u006b\u0065\u0072\u006e\u0065\u006cProfileSaveChanges: z.ZodOptional<z.ZodBoolean>;
70
+ \u006b\u0065\u0072\u006e\u0065\u006cStealth: z.ZodOptional<z.ZodBoolean>;
32
71
  serpIdentity: z.ZodOptional<z.ZodString>;
33
- kernelProxyResolution: z.ZodOptional<z.ZodUnknown>;
72
+ \u006b\u0065\u0072\u006e\u0065\u006cProxyResolution: z.ZodOptional<z.ZodUnknown>;
34
73
  outputDir: z.ZodDefault<z.ZodString>;
35
74
  format: z.ZodDefault<z.ZodEnum<{
36
75
  json: "json";
@@ -57,75 +96,6 @@ declare const HarvestOptionsSchema: z.ZodObject<{
57
96
  softDeadlineMs: z.ZodOptional<z.ZodNumber>;
58
97
  }, z.core.$strip>;
59
98
 
60
- type CanonicalLocationName = string;
61
- type SerpDevice = 'desktop' | 'mobile';
62
- type ProxyMode = 'location' | 'configured' | 'none';
63
- interface KernelLocationProxyTarget {
64
- canonicalLocation: CanonicalLocationName;
65
- level: 'zip' | 'city' | 'state';
66
- country: 'US';
67
- state: string;
68
- city: string;
69
- zip?: string;
70
- cityCandidates: string[];
71
- proxyName: string;
72
- config: {
73
- country: 'US';
74
- state: string;
75
- city?: string;
76
- zip?: string;
77
- };
78
- }
79
- type KernelProxyResolutionSource = 'disabled' | 'location_reused' | 'location_created' | 'isp_created' | 'backup_custom' | 'configured_fallback' | 'unavailable';
80
- interface KernelProxyResolutionDebug {
81
- source: KernelProxyResolutionSource;
82
- proxyType: string;
83
- proxyMode: ProxyMode;
84
- proxyIdPresent: boolean;
85
- proxyIdSuffix: string | null;
86
- target: KernelLocationProxyTarget | null;
87
- error: string | null;
88
- disposable?: boolean;
89
- }
90
- interface HarvestStats {
91
- seed: string;
92
- totalQuestions: number;
93
- maxDepthReached: number;
94
- durationMs: number;
95
- errorCount: number;
96
- }
97
- type HarvestCompletionStatus = 'paa_found' | 'no_paa' | 'serp_only';
98
- /**
99
- * A completed browser attempt can still be unsuitable for programmatic use.
100
- * This is deliberately separate from completionStatus so a valid no-PAA
101
- * response with organic results is not confused with an unreadable SERP.
102
- */
103
- type HarvestResultQuality = 'complete' | 'partial' | 'degraded';
104
- type HarvestDegradationReason = 'empty_primary_serp';
105
- type HarvestDiagnosticWarningCode = 'short_videos_captcha_skipped' | 'paa_partial';
106
- type HarvestDiagnosticWarningSurface = 'short_videos' | 'paa';
107
- interface HarvestDiagnosticWarning {
108
- code: HarvestDiagnosticWarningCode;
109
- surface: HarvestDiagnosticWarningSurface;
110
- message: string;
111
- retryable: boolean;
112
- }
113
- interface HarvestCompleteness {
114
- paaWithoutAnswer: number;
115
- paaWithoutSource: number;
116
- paaAnswersRecovered: number;
117
- aioShareCaptured: boolean | null;
118
- paaRequested?: number;
119
- paaReturned?: number;
120
- paaUnique?: number;
121
- }
122
- type HarvestInterruptionCode = 'browser_session_interrupted' | 'request_aborted' | 'timeout';
123
- type HarvestInterruptionReasonSource = 'client_runtime' | 'extractor';
124
- interface HarvestInterruptionDiagnostic {
125
- code: HarvestInterruptionCode;
126
- reasonSource: HarvestInterruptionReasonSource;
127
- message: string | null;
128
- }
129
99
  type PaaDiscoveryStatus = 'in_progress' | 'target_reached' | 'frontier_exhausted' | 'no_paa_observed' | 'interrupted' | 'recovery_exhausted';
130
100
  type PaaTargetStatus = 'met' | 'not_met';
131
101
  type PaaMaterialStatus = 'complete' | 'partial';
@@ -185,15 +155,6 @@ interface PaaInteractionObservation {
185
155
  errorCode: 'click_failed' | 'click_ack_timeout' | 'control_missing' | 'confirmation_timeout' | null;
186
156
  }
187
157
  type PaaProgressPhase = 'serp_captured' | 'discovered' | 'click_observed' | 'answer_captured' | 'fill_incomplete' | 'expansion_finished' | 'attempt_ending';
188
- /** Observed organic pagination outcome; page two never expands a separate PAA graph. */
189
- interface HarvestPaginationDiagnostics {
190
- requestedPages: 1 | 2;
191
- capturedPages: 1 | 2;
192
- page2Status: 'not_requested' | 'not_attempted' | 'captured' | 'unavailable' | 'failed';
193
- page1OrganicCount: number;
194
- page2OrganicCount: number;
195
- failureCode?: 'missing_next' | 'invalid_next' | 'empty_page' | 'captcha' | 'timeout' | 'navigation_error' | 'unsupported_driver';
196
- }
197
158
  /** Material already observed before PAA expansion completes. It is checkpointed
198
159
  * so a worker interruption cannot discard the SERP that led to the PAA graph. */
199
160
  interface PaaProgressMaterial {
@@ -218,6 +179,119 @@ interface PaaProgressSnapshot {
218
179
  material?: PaaProgressMaterial;
219
180
  interactions?: PaaInteractionObservation[];
220
181
  }
182
+ interface PaaSource extends SerpOutboundLink {
183
+ title: string | null;
184
+ site: string | null;
185
+ }
186
+ interface PAANode {
187
+ questionId: string;
188
+ googleLinkId: string | null;
189
+ googleEvidenceId: string | null;
190
+ question: string;
191
+ answer: string | null;
192
+ sourceTitle: string | null;
193
+ sourceSite: string | null;
194
+ sourceCite: string | null;
195
+ sources: PaaSource[];
196
+ depth: number;
197
+ parentQuestion: string | null;
198
+ children: PAANode[];
199
+ }
200
+ interface FlatRow {
201
+ seed_query: string;
202
+ question_id: string;
203
+ google_link_id: string;
204
+ google_evidence_id: string;
205
+ question: string;
206
+ answer: string;
207
+ source_title: string;
208
+ source_site: string;
209
+ source_cite: string;
210
+ sources: PaaSource[];
211
+ depth: number;
212
+ parent_question: string;
213
+ extracted_at: string;
214
+ }
215
+ type HarvestOptions = z.infer<typeof HarvestOptionsSchema>;
216
+
217
+ type CanonicalLocationName = string;
218
+ type SerpDevice = 'desktop' | 'mobile';
219
+ type ProxyMode = 'location' | 'configured' | 'none';
220
+ interface \u004b\u0065\u0072\u006e\u0065\u006cLocationProxyTarget {
221
+ canonicalLocation: CanonicalLocationName;
222
+ level: 'zip' | 'city' | 'state';
223
+ country: 'US';
224
+ state: string;
225
+ city: string;
226
+ zip?: string;
227
+ cityCandidates: string[];
228
+ proxyName: string;
229
+ config: {
230
+ country: 'US';
231
+ state: string;
232
+ city?: string;
233
+ zip?: string;
234
+ };
235
+ }
236
+ type \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionSource = 'disabled' | 'location_reused' | 'location_created' | 'isp_created' | 'backup_custom' | 'configured_fallback' | 'unavailable';
237
+ interface \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionDebug {
238
+ source: \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionSource;
239
+ proxyType: string;
240
+ proxyMode: ProxyMode;
241
+ proxyIdPresent: boolean;
242
+ proxyIdSuffix: string | null;
243
+ target: \u004b\u0065\u0072\u006e\u0065\u006cLocationProxyTarget | null;
244
+ error: string | null;
245
+ disposable?: boolean;
246
+ }
247
+ interface HarvestStats {
248
+ seed: string;
249
+ totalQuestions: number;
250
+ maxDepthReached: number;
251
+ durationMs: number;
252
+ errorCount: number;
253
+ }
254
+ type HarvestCompletionStatus = 'paa_found' | 'no_paa' | 'serp_only';
255
+ /**
256
+ * A completed browser attempt can still be unsuitable for programmatic use.
257
+ * This is deliberately separate from completionStatus so a valid no-PAA
258
+ * response with organic results is not confused with an unreadable SERP.
259
+ */
260
+ type HarvestResultQuality = 'complete' | 'partial' | 'degraded';
261
+ type HarvestDegradationReason = 'empty_primary_serp' | 'page_two_missing';
262
+ type HarvestDiagnosticWarningCode = 'short_videos_captcha_skipped' | 'paa_partial';
263
+ type HarvestDiagnosticWarningSurface = 'short_videos' | 'paa';
264
+ interface HarvestDiagnosticWarning {
265
+ code: HarvestDiagnosticWarningCode;
266
+ surface: HarvestDiagnosticWarningSurface;
267
+ message: string;
268
+ retryable: boolean;
269
+ }
270
+ interface HarvestCompleteness {
271
+ paaWithoutAnswer: number;
272
+ paaWithoutSource: number;
273
+ paaAnswersRecovered: number;
274
+ aioShareCaptured: boolean | null;
275
+ paaRequested?: number;
276
+ paaReturned?: number;
277
+ paaUnique?: number;
278
+ }
279
+ type HarvestInterruptionCode = 'browser_session_interrupted' | 'request_aborted' | 'timeout';
280
+ type HarvestInterruptionReasonSource = 'client_runtime' | 'extractor';
281
+ interface HarvestInterruptionDiagnostic {
282
+ code: HarvestInterruptionCode;
283
+ reasonSource: HarvestInterruptionReasonSource;
284
+ message: string | null;
285
+ }
286
+ /** Observed organic pagination outcome; page two never expands a separate PAA graph. */
287
+ interface HarvestPaginationDiagnostics {
288
+ requestedPages: 1 | 2;
289
+ capturedPages: 1 | 2;
290
+ page2Status: 'not_requested' | 'not_attempted' | 'captured' | 'unavailable' | 'failed';
291
+ page1OrganicCount: number;
292
+ page2OrganicCount: number;
293
+ failureCode?: 'missing_next' | 'invalid_next' | 'empty_page' | 'captcha' | 'timeout' | 'navigation_error' | 'unsupported_driver';
294
+ }
221
295
  interface HarvestDiagnostics {
222
296
  phaseTimings?: {
223
297
  providerConnectMs?: number;
@@ -265,7 +339,7 @@ interface BrowserNetworkLocationDebug {
265
339
  timezone: string | null;
266
340
  error: string | null;
267
341
  }
268
- interface KernelSessionDebug {
342
+ interface \u004b\u0065\u0072\u006e\u0065\u006cSessionDebug {
269
343
  sessionId: string | null;
270
344
  proxyMode: ProxyMode;
271
345
  requestedProxyIdPresent: boolean;
@@ -277,7 +351,7 @@ interface KernelSessionDebug {
277
351
  retrievedProxyIdMatchesRequested: boolean | null;
278
352
  defaultProxyDisabled: boolean | null;
279
353
  defaultProxyDisableError: string | null;
280
- proxyResolution: KernelProxyResolutionDebug | null;
354
+ proxyResolution: \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionDebug | null;
281
355
  timeoutSeconds: number | null;
282
356
  stealth: boolean | null;
283
357
  profilePresent: boolean | null;
@@ -307,7 +381,7 @@ interface SerpNavigationDebug {
307
381
  redirected: boolean | null;
308
382
  }
309
383
  interface BrowserDebugSnapshot {
310
- kernel: KernelSessionDebug | null;
384
+ \u006b\u0065\u0072\u006e\u0065\u006c: \u004b\u0065\u0072\u006e\u0065\u006cSessionDebug | null;
311
385
  context: BrowserContextDebug | null;
312
386
  networkLocation: BrowserNetworkLocationDebug | null;
313
387
  serpNavigation: SerpNavigationDebug | null;
@@ -358,39 +432,6 @@ interface SerpOutboundLink {
358
432
  linkType: SerpLinkType;
359
433
  resolutionStatus: SerpLinkResolutionStatus;
360
434
  }
361
- interface PaaSource extends SerpOutboundLink {
362
- title: string | null;
363
- site: string | null;
364
- }
365
- interface PAANode {
366
- questionId: string;
367
- googleLinkId: string | null;
368
- googleEvidenceId: string | null;
369
- question: string;
370
- answer: string | null;
371
- sourceTitle: string | null;
372
- sourceSite: string | null;
373
- sourceCite: string | null;
374
- sources: PaaSource[];
375
- depth: number;
376
- parentQuestion: string | null;
377
- children: PAANode[];
378
- }
379
- interface FlatRow {
380
- seed_query: string;
381
- question_id: string;
382
- google_link_id: string;
383
- google_evidence_id: string;
384
- question: string;
385
- answer: string;
386
- source_title: string;
387
- source_site: string;
388
- source_cite: string;
389
- sources: PaaSource[];
390
- depth: number;
391
- parent_question: string;
392
- extracted_at: string;
393
- }
394
435
  interface VideoResult {
395
436
  type: 'video' | 'short_video';
396
437
  title: string;
@@ -495,9 +536,9 @@ interface HarvestResult {
495
536
  organicResults: OrganicResult[];
496
537
  localPack: LocalPackBusiness[];
497
538
  entityIds: EntityIds;
539
+ fullSerpFeatures?: FullSerpFeatures;
498
540
  stats: HarvestStats;
499
541
  }
500
- type HarvestOptions = z.infer<typeof HarvestOptionsSchema>;
501
542
 
502
543
  declare function harvest(rawOptions: unknown): Promise<HarvestResult>;
503
544