mcp-scraper 0.92.1 → 0.93.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +45 -26
- package/README.md +4 -4
- package/THIRD_PARTY_NOTICES.html +203 -0
- package/dist/bin/api-server.js +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-core.js +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.js +1 -1
- package/dist/bin/paa-harvest.js +1 -1
- package/dist/chunk-2SP57VCG.js +15 -15
- package/dist/{chunk-QHHOKLQY.js → chunk-5JKBFNYF.js} +1 -1
- package/dist/{chunk-ZJZMDWDI.js → chunk-5QYW4CSJ.js} +5 -5
- package/dist/{chunk-YXUPFECS.js → chunk-AFFMCO7R.js} +2 -2
- package/dist/chunk-AYQSKD7D.js +1 -0
- package/dist/chunk-DT2FYN6N.js +1 -0
- package/dist/{chunk-TCLSPE3P.js → chunk-IJF4VXJ7.js} +1 -1
- package/dist/chunk-KPQM4UDJ.js +6 -0
- package/dist/chunk-M5QHXNFZ.js +3 -3
- package/dist/{chunk-ZMNUI5LB.js → chunk-M5VZVMPC.js} +8 -8
- package/dist/{chunk-T2IJVEHG.js → chunk-NLKA6SHC.js} +1 -1
- package/dist/{chunk-QXAXPIHM.js → chunk-OD2WLHEN.js} +243 -240
- package/dist/chunk-ORB4RHCK.js +4 -4
- package/dist/chunk-SH5KB4P7.js +1 -0
- package/dist/chunk-UN6FVDJQ.js +1 -0
- package/dist/{chunk-JHMI6HEO.js → chunk-W2T4NTCE.js} +1 -1
- package/dist/chunk-WJ4XFLS4.js +15 -15
- package/dist/{chunk-GPZ7ITF6.js → chunk-X6RD3Y4W.js} +1 -1
- package/dist/chunk-Y6MKMSOC.js +1 -0
- package/dist/{extract-bundle-5QTPVIWO.js → extract-bundle-C3N6E6V7.js} +1 -1
- package/dist/{gmail-service-EOC7KJSK.js → gmail-service-6V5MFBYH.js} +1 -1
- package/dist/index.cjs +57 -57
- package/dist/index.d.cts +162 -121
- package/dist/index.d.ts +162 -121
- package/dist/index.js +6 -6
- package/dist/{server-NAQTBNPX.js → server-KWGXYWYO.js} +71 -71
- package/dist/{site-extract-repository-U4G25TR2.js → site-extract-repository-2LFDP6FZ.js} +1 -1
- package/dist/{stripe-event-worker-KW3DR6GB.js → stripe-event-worker-IJXA6T37.js} +1 -1
- package/dist/worker-ASXALAY7.js +1 -0
- package/package.json +17 -136
- package/dist/chunk-2BN2TB3U.js +0 -1
- package/dist/chunk-2TZBO52D.js +0 -1
- package/dist/chunk-4FROKQJN.js +0 -1
- package/dist/chunk-GCLSGYYE.js +0 -1
- package/dist/chunk-K6MN7UED.js +0 -2
- package/dist/chunk-RECKUOEF.js +0 -1
- package/dist/chunk-RSPA4IWH.js +0 -1
- package/dist/worker-MSLXSR4M.js +0 -1
package/dist/index.d.cts
CHANGED
|
@@ -1,5 +1,44 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
|
|
3
|
+
declare const SERP_FEATURE_COMPLETENESS_STATES: readonly ["observed_present", "observed_absent", "incomplete", "not_requested", "unknown", "unsupported"];
|
|
4
|
+
type SerpFeatureCompletenessState = typeof SERP_FEATURE_COMPLETENESS_STATES[number];
|
|
5
|
+
|
|
6
|
+
interface SerpPreviewQuestion {
|
|
7
|
+
question: string;
|
|
8
|
+
answer: string | null;
|
|
9
|
+
url: string | null;
|
|
10
|
+
}
|
|
11
|
+
interface SerpFeatureItem {
|
|
12
|
+
title: string | null;
|
|
13
|
+
url: string | null;
|
|
14
|
+
description: string | null;
|
|
15
|
+
position: number | null;
|
|
16
|
+
}
|
|
17
|
+
interface SerpFeatureGroup {
|
|
18
|
+
name: string;
|
|
19
|
+
attributes: Record<string, string | number | boolean | null>;
|
|
20
|
+
items: SerpFeatureItem[];
|
|
21
|
+
}
|
|
22
|
+
interface FullSerpPageFeatures {
|
|
23
|
+
page: 1 | 2;
|
|
24
|
+
status: Record<string, SerpFeatureCompletenessState>;
|
|
25
|
+
paaPreview: SerpPreviewQuestion[];
|
|
26
|
+
additional: SerpFeatureGroup[];
|
|
27
|
+
localPack: LocalPackBusiness[];
|
|
28
|
+
forums: ForumResult[];
|
|
29
|
+
videos: VideoResult[];
|
|
30
|
+
whatPeopleSaying: WhatPeopleSayingCard[];
|
|
31
|
+
aiOverview: AIOverviewResult;
|
|
32
|
+
aiMode: AIModeResult;
|
|
33
|
+
entityIds: EntityIds;
|
|
34
|
+
}
|
|
35
|
+
interface FullSerpFeatures {
|
|
36
|
+
status: Record<string, SerpFeatureCompletenessState>;
|
|
37
|
+
paaPreview: SerpPreviewQuestion[];
|
|
38
|
+
additional: SerpFeatureGroup[];
|
|
39
|
+
pages: FullSerpPageFeatures[];
|
|
40
|
+
}
|
|
41
|
+
|
|
3
42
|
declare const HarvestOptionsSchema: z.ZodObject<{
|
|
4
43
|
query: z.ZodString;
|
|
5
44
|
location: z.ZodOptional<z.ZodString>;
|
|
@@ -24,13 +63,13 @@ declare const HarvestOptionsSchema: z.ZodObject<{
|
|
|
24
63
|
headless: z.ZodDefault<z.ZodBoolean>;
|
|
25
64
|
profileDir: z.ZodOptional<z.ZodString>;
|
|
26
65
|
proxy: z.ZodOptional<z.ZodString>;
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
66
|
+
\u006b\u0065\u0072\u006e\u0065\u006cApiKey: z.ZodOptional<z.ZodString>;
|
|
67
|
+
\u006b\u0065\u0072\u006e\u0065\u006cProxyId: z.ZodOptional<z.ZodString>;
|
|
68
|
+
\u006b\u0065\u0072\u006e\u0065\u006cProfileName: z.ZodOptional<z.ZodString>;
|
|
69
|
+
\u006b\u0065\u0072\u006e\u0065\u006cProfileSaveChanges: z.ZodOptional<z.ZodBoolean>;
|
|
70
|
+
\u006b\u0065\u0072\u006e\u0065\u006cStealth: z.ZodOptional<z.ZodBoolean>;
|
|
32
71
|
serpIdentity: z.ZodOptional<z.ZodString>;
|
|
33
|
-
|
|
72
|
+
\u006b\u0065\u0072\u006e\u0065\u006cProxyResolution: z.ZodOptional<z.ZodUnknown>;
|
|
34
73
|
outputDir: z.ZodDefault<z.ZodString>;
|
|
35
74
|
format: z.ZodDefault<z.ZodEnum<{
|
|
36
75
|
json: "json";
|
|
@@ -57,75 +96,6 @@ declare const HarvestOptionsSchema: z.ZodObject<{
|
|
|
57
96
|
softDeadlineMs: z.ZodOptional<z.ZodNumber>;
|
|
58
97
|
}, z.core.$strip>;
|
|
59
98
|
|
|
60
|
-
type CanonicalLocationName = string;
|
|
61
|
-
type SerpDevice = 'desktop' | 'mobile';
|
|
62
|
-
type ProxyMode = 'location' | 'configured' | 'none';
|
|
63
|
-
interface KernelLocationProxyTarget {
|
|
64
|
-
canonicalLocation: CanonicalLocationName;
|
|
65
|
-
level: 'zip' | 'city' | 'state';
|
|
66
|
-
country: 'US';
|
|
67
|
-
state: string;
|
|
68
|
-
city: string;
|
|
69
|
-
zip?: string;
|
|
70
|
-
cityCandidates: string[];
|
|
71
|
-
proxyName: string;
|
|
72
|
-
config: {
|
|
73
|
-
country: 'US';
|
|
74
|
-
state: string;
|
|
75
|
-
city?: string;
|
|
76
|
-
zip?: string;
|
|
77
|
-
};
|
|
78
|
-
}
|
|
79
|
-
type KernelProxyResolutionSource = 'disabled' | 'location_reused' | 'location_created' | 'isp_created' | 'backup_custom' | 'configured_fallback' | 'unavailable';
|
|
80
|
-
interface KernelProxyResolutionDebug {
|
|
81
|
-
source: KernelProxyResolutionSource;
|
|
82
|
-
proxyType: string;
|
|
83
|
-
proxyMode: ProxyMode;
|
|
84
|
-
proxyIdPresent: boolean;
|
|
85
|
-
proxyIdSuffix: string | null;
|
|
86
|
-
target: KernelLocationProxyTarget | null;
|
|
87
|
-
error: string | null;
|
|
88
|
-
disposable?: boolean;
|
|
89
|
-
}
|
|
90
|
-
interface HarvestStats {
|
|
91
|
-
seed: string;
|
|
92
|
-
totalQuestions: number;
|
|
93
|
-
maxDepthReached: number;
|
|
94
|
-
durationMs: number;
|
|
95
|
-
errorCount: number;
|
|
96
|
-
}
|
|
97
|
-
type HarvestCompletionStatus = 'paa_found' | 'no_paa' | 'serp_only';
|
|
98
|
-
/**
|
|
99
|
-
* A completed browser attempt can still be unsuitable for programmatic use.
|
|
100
|
-
* This is deliberately separate from completionStatus so a valid no-PAA
|
|
101
|
-
* response with organic results is not confused with an unreadable SERP.
|
|
102
|
-
*/
|
|
103
|
-
type HarvestResultQuality = 'complete' | 'partial' | 'degraded';
|
|
104
|
-
type HarvestDegradationReason = 'empty_primary_serp';
|
|
105
|
-
type HarvestDiagnosticWarningCode = 'short_videos_captcha_skipped' | 'paa_partial';
|
|
106
|
-
type HarvestDiagnosticWarningSurface = 'short_videos' | 'paa';
|
|
107
|
-
interface HarvestDiagnosticWarning {
|
|
108
|
-
code: HarvestDiagnosticWarningCode;
|
|
109
|
-
surface: HarvestDiagnosticWarningSurface;
|
|
110
|
-
message: string;
|
|
111
|
-
retryable: boolean;
|
|
112
|
-
}
|
|
113
|
-
interface HarvestCompleteness {
|
|
114
|
-
paaWithoutAnswer: number;
|
|
115
|
-
paaWithoutSource: number;
|
|
116
|
-
paaAnswersRecovered: number;
|
|
117
|
-
aioShareCaptured: boolean | null;
|
|
118
|
-
paaRequested?: number;
|
|
119
|
-
paaReturned?: number;
|
|
120
|
-
paaUnique?: number;
|
|
121
|
-
}
|
|
122
|
-
type HarvestInterruptionCode = 'browser_session_interrupted' | 'request_aborted' | 'timeout';
|
|
123
|
-
type HarvestInterruptionReasonSource = 'client_runtime' | 'extractor';
|
|
124
|
-
interface HarvestInterruptionDiagnostic {
|
|
125
|
-
code: HarvestInterruptionCode;
|
|
126
|
-
reasonSource: HarvestInterruptionReasonSource;
|
|
127
|
-
message: string | null;
|
|
128
|
-
}
|
|
129
99
|
type PaaDiscoveryStatus = 'in_progress' | 'target_reached' | 'frontier_exhausted' | 'no_paa_observed' | 'interrupted' | 'recovery_exhausted';
|
|
130
100
|
type PaaTargetStatus = 'met' | 'not_met';
|
|
131
101
|
type PaaMaterialStatus = 'complete' | 'partial';
|
|
@@ -185,15 +155,6 @@ interface PaaInteractionObservation {
|
|
|
185
155
|
errorCode: 'click_failed' | 'click_ack_timeout' | 'control_missing' | 'confirmation_timeout' | null;
|
|
186
156
|
}
|
|
187
157
|
type PaaProgressPhase = 'serp_captured' | 'discovered' | 'click_observed' | 'answer_captured' | 'fill_incomplete' | 'expansion_finished' | 'attempt_ending';
|
|
188
|
-
/** Observed organic pagination outcome; page two never expands a separate PAA graph. */
|
|
189
|
-
interface HarvestPaginationDiagnostics {
|
|
190
|
-
requestedPages: 1 | 2;
|
|
191
|
-
capturedPages: 1 | 2;
|
|
192
|
-
page2Status: 'not_requested' | 'not_attempted' | 'captured' | 'unavailable' | 'failed';
|
|
193
|
-
page1OrganicCount: number;
|
|
194
|
-
page2OrganicCount: number;
|
|
195
|
-
failureCode?: 'missing_next' | 'invalid_next' | 'empty_page' | 'captcha' | 'timeout' | 'navigation_error' | 'unsupported_driver';
|
|
196
|
-
}
|
|
197
158
|
/** Material already observed before PAA expansion completes. It is checkpointed
|
|
198
159
|
* so a worker interruption cannot discard the SERP that led to the PAA graph. */
|
|
199
160
|
interface PaaProgressMaterial {
|
|
@@ -218,6 +179,119 @@ interface PaaProgressSnapshot {
|
|
|
218
179
|
material?: PaaProgressMaterial;
|
|
219
180
|
interactions?: PaaInteractionObservation[];
|
|
220
181
|
}
|
|
182
|
+
interface PaaSource extends SerpOutboundLink {
|
|
183
|
+
title: string | null;
|
|
184
|
+
site: string | null;
|
|
185
|
+
}
|
|
186
|
+
interface PAANode {
|
|
187
|
+
questionId: string;
|
|
188
|
+
googleLinkId: string | null;
|
|
189
|
+
googleEvidenceId: string | null;
|
|
190
|
+
question: string;
|
|
191
|
+
answer: string | null;
|
|
192
|
+
sourceTitle: string | null;
|
|
193
|
+
sourceSite: string | null;
|
|
194
|
+
sourceCite: string | null;
|
|
195
|
+
sources: PaaSource[];
|
|
196
|
+
depth: number;
|
|
197
|
+
parentQuestion: string | null;
|
|
198
|
+
children: PAANode[];
|
|
199
|
+
}
|
|
200
|
+
interface FlatRow {
|
|
201
|
+
seed_query: string;
|
|
202
|
+
question_id: string;
|
|
203
|
+
google_link_id: string;
|
|
204
|
+
google_evidence_id: string;
|
|
205
|
+
question: string;
|
|
206
|
+
answer: string;
|
|
207
|
+
source_title: string;
|
|
208
|
+
source_site: string;
|
|
209
|
+
source_cite: string;
|
|
210
|
+
sources: PaaSource[];
|
|
211
|
+
depth: number;
|
|
212
|
+
parent_question: string;
|
|
213
|
+
extracted_at: string;
|
|
214
|
+
}
|
|
215
|
+
type HarvestOptions = z.infer<typeof HarvestOptionsSchema>;
|
|
216
|
+
|
|
217
|
+
type CanonicalLocationName = string;
|
|
218
|
+
type SerpDevice = 'desktop' | 'mobile';
|
|
219
|
+
type ProxyMode = 'location' | 'configured' | 'none';
|
|
220
|
+
interface \u004b\u0065\u0072\u006e\u0065\u006cLocationProxyTarget {
|
|
221
|
+
canonicalLocation: CanonicalLocationName;
|
|
222
|
+
level: 'zip' | 'city' | 'state';
|
|
223
|
+
country: 'US';
|
|
224
|
+
state: string;
|
|
225
|
+
city: string;
|
|
226
|
+
zip?: string;
|
|
227
|
+
cityCandidates: string[];
|
|
228
|
+
proxyName: string;
|
|
229
|
+
config: {
|
|
230
|
+
country: 'US';
|
|
231
|
+
state: string;
|
|
232
|
+
city?: string;
|
|
233
|
+
zip?: string;
|
|
234
|
+
};
|
|
235
|
+
}
|
|
236
|
+
type \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionSource = 'disabled' | 'location_reused' | 'location_created' | 'isp_created' | 'backup_custom' | 'configured_fallback' | 'unavailable';
|
|
237
|
+
interface \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionDebug {
|
|
238
|
+
source: \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionSource;
|
|
239
|
+
proxyType: string;
|
|
240
|
+
proxyMode: ProxyMode;
|
|
241
|
+
proxyIdPresent: boolean;
|
|
242
|
+
proxyIdSuffix: string | null;
|
|
243
|
+
target: \u004b\u0065\u0072\u006e\u0065\u006cLocationProxyTarget | null;
|
|
244
|
+
error: string | null;
|
|
245
|
+
disposable?: boolean;
|
|
246
|
+
}
|
|
247
|
+
interface HarvestStats {
|
|
248
|
+
seed: string;
|
|
249
|
+
totalQuestions: number;
|
|
250
|
+
maxDepthReached: number;
|
|
251
|
+
durationMs: number;
|
|
252
|
+
errorCount: number;
|
|
253
|
+
}
|
|
254
|
+
type HarvestCompletionStatus = 'paa_found' | 'no_paa' | 'serp_only';
|
|
255
|
+
/**
|
|
256
|
+
* A completed browser attempt can still be unsuitable for programmatic use.
|
|
257
|
+
* This is deliberately separate from completionStatus so a valid no-PAA
|
|
258
|
+
* response with organic results is not confused with an unreadable SERP.
|
|
259
|
+
*/
|
|
260
|
+
type HarvestResultQuality = 'complete' | 'partial' | 'degraded';
|
|
261
|
+
type HarvestDegradationReason = 'empty_primary_serp' | 'page_two_missing';
|
|
262
|
+
type HarvestDiagnosticWarningCode = 'short_videos_captcha_skipped' | 'paa_partial';
|
|
263
|
+
type HarvestDiagnosticWarningSurface = 'short_videos' | 'paa';
|
|
264
|
+
interface HarvestDiagnosticWarning {
|
|
265
|
+
code: HarvestDiagnosticWarningCode;
|
|
266
|
+
surface: HarvestDiagnosticWarningSurface;
|
|
267
|
+
message: string;
|
|
268
|
+
retryable: boolean;
|
|
269
|
+
}
|
|
270
|
+
interface HarvestCompleteness {
|
|
271
|
+
paaWithoutAnswer: number;
|
|
272
|
+
paaWithoutSource: number;
|
|
273
|
+
paaAnswersRecovered: number;
|
|
274
|
+
aioShareCaptured: boolean | null;
|
|
275
|
+
paaRequested?: number;
|
|
276
|
+
paaReturned?: number;
|
|
277
|
+
paaUnique?: number;
|
|
278
|
+
}
|
|
279
|
+
type HarvestInterruptionCode = 'browser_session_interrupted' | 'request_aborted' | 'timeout';
|
|
280
|
+
type HarvestInterruptionReasonSource = 'client_runtime' | 'extractor';
|
|
281
|
+
interface HarvestInterruptionDiagnostic {
|
|
282
|
+
code: HarvestInterruptionCode;
|
|
283
|
+
reasonSource: HarvestInterruptionReasonSource;
|
|
284
|
+
message: string | null;
|
|
285
|
+
}
|
|
286
|
+
/** Observed organic pagination outcome; page two never expands a separate PAA graph. */
|
|
287
|
+
interface HarvestPaginationDiagnostics {
|
|
288
|
+
requestedPages: 1 | 2;
|
|
289
|
+
capturedPages: 1 | 2;
|
|
290
|
+
page2Status: 'not_requested' | 'not_attempted' | 'captured' | 'unavailable' | 'failed';
|
|
291
|
+
page1OrganicCount: number;
|
|
292
|
+
page2OrganicCount: number;
|
|
293
|
+
failureCode?: 'missing_next' | 'invalid_next' | 'empty_page' | 'captcha' | 'timeout' | 'navigation_error' | 'unsupported_driver';
|
|
294
|
+
}
|
|
221
295
|
interface HarvestDiagnostics {
|
|
222
296
|
phaseTimings?: {
|
|
223
297
|
providerConnectMs?: number;
|
|
@@ -265,7 +339,7 @@ interface BrowserNetworkLocationDebug {
|
|
|
265
339
|
timezone: string | null;
|
|
266
340
|
error: string | null;
|
|
267
341
|
}
|
|
268
|
-
interface
|
|
342
|
+
interface \u004b\u0065\u0072\u006e\u0065\u006cSessionDebug {
|
|
269
343
|
sessionId: string | null;
|
|
270
344
|
proxyMode: ProxyMode;
|
|
271
345
|
requestedProxyIdPresent: boolean;
|
|
@@ -277,7 +351,7 @@ interface KernelSessionDebug {
|
|
|
277
351
|
retrievedProxyIdMatchesRequested: boolean | null;
|
|
278
352
|
defaultProxyDisabled: boolean | null;
|
|
279
353
|
defaultProxyDisableError: string | null;
|
|
280
|
-
proxyResolution:
|
|
354
|
+
proxyResolution: \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionDebug | null;
|
|
281
355
|
timeoutSeconds: number | null;
|
|
282
356
|
stealth: boolean | null;
|
|
283
357
|
profilePresent: boolean | null;
|
|
@@ -307,7 +381,7 @@ interface SerpNavigationDebug {
|
|
|
307
381
|
redirected: boolean | null;
|
|
308
382
|
}
|
|
309
383
|
interface BrowserDebugSnapshot {
|
|
310
|
-
|
|
384
|
+
\u006b\u0065\u0072\u006e\u0065\u006c: \u004b\u0065\u0072\u006e\u0065\u006cSessionDebug | null;
|
|
311
385
|
context: BrowserContextDebug | null;
|
|
312
386
|
networkLocation: BrowserNetworkLocationDebug | null;
|
|
313
387
|
serpNavigation: SerpNavigationDebug | null;
|
|
@@ -358,39 +432,6 @@ interface SerpOutboundLink {
|
|
|
358
432
|
linkType: SerpLinkType;
|
|
359
433
|
resolutionStatus: SerpLinkResolutionStatus;
|
|
360
434
|
}
|
|
361
|
-
interface PaaSource extends SerpOutboundLink {
|
|
362
|
-
title: string | null;
|
|
363
|
-
site: string | null;
|
|
364
|
-
}
|
|
365
|
-
interface PAANode {
|
|
366
|
-
questionId: string;
|
|
367
|
-
googleLinkId: string | null;
|
|
368
|
-
googleEvidenceId: string | null;
|
|
369
|
-
question: string;
|
|
370
|
-
answer: string | null;
|
|
371
|
-
sourceTitle: string | null;
|
|
372
|
-
sourceSite: string | null;
|
|
373
|
-
sourceCite: string | null;
|
|
374
|
-
sources: PaaSource[];
|
|
375
|
-
depth: number;
|
|
376
|
-
parentQuestion: string | null;
|
|
377
|
-
children: PAANode[];
|
|
378
|
-
}
|
|
379
|
-
interface FlatRow {
|
|
380
|
-
seed_query: string;
|
|
381
|
-
question_id: string;
|
|
382
|
-
google_link_id: string;
|
|
383
|
-
google_evidence_id: string;
|
|
384
|
-
question: string;
|
|
385
|
-
answer: string;
|
|
386
|
-
source_title: string;
|
|
387
|
-
source_site: string;
|
|
388
|
-
source_cite: string;
|
|
389
|
-
sources: PaaSource[];
|
|
390
|
-
depth: number;
|
|
391
|
-
parent_question: string;
|
|
392
|
-
extracted_at: string;
|
|
393
|
-
}
|
|
394
435
|
interface VideoResult {
|
|
395
436
|
type: 'video' | 'short_video';
|
|
396
437
|
title: string;
|
|
@@ -495,9 +536,9 @@ interface HarvestResult {
|
|
|
495
536
|
organicResults: OrganicResult[];
|
|
496
537
|
localPack: LocalPackBusiness[];
|
|
497
538
|
entityIds: EntityIds;
|
|
539
|
+
fullSerpFeatures?: FullSerpFeatures;
|
|
498
540
|
stats: HarvestStats;
|
|
499
541
|
}
|
|
500
|
-
type HarvestOptions = z.infer<typeof HarvestOptionsSchema>;
|
|
501
542
|
|
|
502
543
|
declare function harvest(rawOptions: unknown): Promise<HarvestResult>;
|
|
503
544
|
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,44 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
|
|
3
|
+
declare const SERP_FEATURE_COMPLETENESS_STATES: readonly ["observed_present", "observed_absent", "incomplete", "not_requested", "unknown", "unsupported"];
|
|
4
|
+
type SerpFeatureCompletenessState = typeof SERP_FEATURE_COMPLETENESS_STATES[number];
|
|
5
|
+
|
|
6
|
+
interface SerpPreviewQuestion {
|
|
7
|
+
question: string;
|
|
8
|
+
answer: string | null;
|
|
9
|
+
url: string | null;
|
|
10
|
+
}
|
|
11
|
+
interface SerpFeatureItem {
|
|
12
|
+
title: string | null;
|
|
13
|
+
url: string | null;
|
|
14
|
+
description: string | null;
|
|
15
|
+
position: number | null;
|
|
16
|
+
}
|
|
17
|
+
interface SerpFeatureGroup {
|
|
18
|
+
name: string;
|
|
19
|
+
attributes: Record<string, string | number | boolean | null>;
|
|
20
|
+
items: SerpFeatureItem[];
|
|
21
|
+
}
|
|
22
|
+
interface FullSerpPageFeatures {
|
|
23
|
+
page: 1 | 2;
|
|
24
|
+
status: Record<string, SerpFeatureCompletenessState>;
|
|
25
|
+
paaPreview: SerpPreviewQuestion[];
|
|
26
|
+
additional: SerpFeatureGroup[];
|
|
27
|
+
localPack: LocalPackBusiness[];
|
|
28
|
+
forums: ForumResult[];
|
|
29
|
+
videos: VideoResult[];
|
|
30
|
+
whatPeopleSaying: WhatPeopleSayingCard[];
|
|
31
|
+
aiOverview: AIOverviewResult;
|
|
32
|
+
aiMode: AIModeResult;
|
|
33
|
+
entityIds: EntityIds;
|
|
34
|
+
}
|
|
35
|
+
interface FullSerpFeatures {
|
|
36
|
+
status: Record<string, SerpFeatureCompletenessState>;
|
|
37
|
+
paaPreview: SerpPreviewQuestion[];
|
|
38
|
+
additional: SerpFeatureGroup[];
|
|
39
|
+
pages: FullSerpPageFeatures[];
|
|
40
|
+
}
|
|
41
|
+
|
|
3
42
|
declare const HarvestOptionsSchema: z.ZodObject<{
|
|
4
43
|
query: z.ZodString;
|
|
5
44
|
location: z.ZodOptional<z.ZodString>;
|
|
@@ -24,13 +63,13 @@ declare const HarvestOptionsSchema: z.ZodObject<{
|
|
|
24
63
|
headless: z.ZodDefault<z.ZodBoolean>;
|
|
25
64
|
profileDir: z.ZodOptional<z.ZodString>;
|
|
26
65
|
proxy: z.ZodOptional<z.ZodString>;
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
66
|
+
\u006b\u0065\u0072\u006e\u0065\u006cApiKey: z.ZodOptional<z.ZodString>;
|
|
67
|
+
\u006b\u0065\u0072\u006e\u0065\u006cProxyId: z.ZodOptional<z.ZodString>;
|
|
68
|
+
\u006b\u0065\u0072\u006e\u0065\u006cProfileName: z.ZodOptional<z.ZodString>;
|
|
69
|
+
\u006b\u0065\u0072\u006e\u0065\u006cProfileSaveChanges: z.ZodOptional<z.ZodBoolean>;
|
|
70
|
+
\u006b\u0065\u0072\u006e\u0065\u006cStealth: z.ZodOptional<z.ZodBoolean>;
|
|
32
71
|
serpIdentity: z.ZodOptional<z.ZodString>;
|
|
33
|
-
|
|
72
|
+
\u006b\u0065\u0072\u006e\u0065\u006cProxyResolution: z.ZodOptional<z.ZodUnknown>;
|
|
34
73
|
outputDir: z.ZodDefault<z.ZodString>;
|
|
35
74
|
format: z.ZodDefault<z.ZodEnum<{
|
|
36
75
|
json: "json";
|
|
@@ -57,75 +96,6 @@ declare const HarvestOptionsSchema: z.ZodObject<{
|
|
|
57
96
|
softDeadlineMs: z.ZodOptional<z.ZodNumber>;
|
|
58
97
|
}, z.core.$strip>;
|
|
59
98
|
|
|
60
|
-
type CanonicalLocationName = string;
|
|
61
|
-
type SerpDevice = 'desktop' | 'mobile';
|
|
62
|
-
type ProxyMode = 'location' | 'configured' | 'none';
|
|
63
|
-
interface KernelLocationProxyTarget {
|
|
64
|
-
canonicalLocation: CanonicalLocationName;
|
|
65
|
-
level: 'zip' | 'city' | 'state';
|
|
66
|
-
country: 'US';
|
|
67
|
-
state: string;
|
|
68
|
-
city: string;
|
|
69
|
-
zip?: string;
|
|
70
|
-
cityCandidates: string[];
|
|
71
|
-
proxyName: string;
|
|
72
|
-
config: {
|
|
73
|
-
country: 'US';
|
|
74
|
-
state: string;
|
|
75
|
-
city?: string;
|
|
76
|
-
zip?: string;
|
|
77
|
-
};
|
|
78
|
-
}
|
|
79
|
-
type KernelProxyResolutionSource = 'disabled' | 'location_reused' | 'location_created' | 'isp_created' | 'backup_custom' | 'configured_fallback' | 'unavailable';
|
|
80
|
-
interface KernelProxyResolutionDebug {
|
|
81
|
-
source: KernelProxyResolutionSource;
|
|
82
|
-
proxyType: string;
|
|
83
|
-
proxyMode: ProxyMode;
|
|
84
|
-
proxyIdPresent: boolean;
|
|
85
|
-
proxyIdSuffix: string | null;
|
|
86
|
-
target: KernelLocationProxyTarget | null;
|
|
87
|
-
error: string | null;
|
|
88
|
-
disposable?: boolean;
|
|
89
|
-
}
|
|
90
|
-
interface HarvestStats {
|
|
91
|
-
seed: string;
|
|
92
|
-
totalQuestions: number;
|
|
93
|
-
maxDepthReached: number;
|
|
94
|
-
durationMs: number;
|
|
95
|
-
errorCount: number;
|
|
96
|
-
}
|
|
97
|
-
type HarvestCompletionStatus = 'paa_found' | 'no_paa' | 'serp_only';
|
|
98
|
-
/**
|
|
99
|
-
* A completed browser attempt can still be unsuitable for programmatic use.
|
|
100
|
-
* This is deliberately separate from completionStatus so a valid no-PAA
|
|
101
|
-
* response with organic results is not confused with an unreadable SERP.
|
|
102
|
-
*/
|
|
103
|
-
type HarvestResultQuality = 'complete' | 'partial' | 'degraded';
|
|
104
|
-
type HarvestDegradationReason = 'empty_primary_serp';
|
|
105
|
-
type HarvestDiagnosticWarningCode = 'short_videos_captcha_skipped' | 'paa_partial';
|
|
106
|
-
type HarvestDiagnosticWarningSurface = 'short_videos' | 'paa';
|
|
107
|
-
interface HarvestDiagnosticWarning {
|
|
108
|
-
code: HarvestDiagnosticWarningCode;
|
|
109
|
-
surface: HarvestDiagnosticWarningSurface;
|
|
110
|
-
message: string;
|
|
111
|
-
retryable: boolean;
|
|
112
|
-
}
|
|
113
|
-
interface HarvestCompleteness {
|
|
114
|
-
paaWithoutAnswer: number;
|
|
115
|
-
paaWithoutSource: number;
|
|
116
|
-
paaAnswersRecovered: number;
|
|
117
|
-
aioShareCaptured: boolean | null;
|
|
118
|
-
paaRequested?: number;
|
|
119
|
-
paaReturned?: number;
|
|
120
|
-
paaUnique?: number;
|
|
121
|
-
}
|
|
122
|
-
type HarvestInterruptionCode = 'browser_session_interrupted' | 'request_aborted' | 'timeout';
|
|
123
|
-
type HarvestInterruptionReasonSource = 'client_runtime' | 'extractor';
|
|
124
|
-
interface HarvestInterruptionDiagnostic {
|
|
125
|
-
code: HarvestInterruptionCode;
|
|
126
|
-
reasonSource: HarvestInterruptionReasonSource;
|
|
127
|
-
message: string | null;
|
|
128
|
-
}
|
|
129
99
|
type PaaDiscoveryStatus = 'in_progress' | 'target_reached' | 'frontier_exhausted' | 'no_paa_observed' | 'interrupted' | 'recovery_exhausted';
|
|
130
100
|
type PaaTargetStatus = 'met' | 'not_met';
|
|
131
101
|
type PaaMaterialStatus = 'complete' | 'partial';
|
|
@@ -185,15 +155,6 @@ interface PaaInteractionObservation {
|
|
|
185
155
|
errorCode: 'click_failed' | 'click_ack_timeout' | 'control_missing' | 'confirmation_timeout' | null;
|
|
186
156
|
}
|
|
187
157
|
type PaaProgressPhase = 'serp_captured' | 'discovered' | 'click_observed' | 'answer_captured' | 'fill_incomplete' | 'expansion_finished' | 'attempt_ending';
|
|
188
|
-
/** Observed organic pagination outcome; page two never expands a separate PAA graph. */
|
|
189
|
-
interface HarvestPaginationDiagnostics {
|
|
190
|
-
requestedPages: 1 | 2;
|
|
191
|
-
capturedPages: 1 | 2;
|
|
192
|
-
page2Status: 'not_requested' | 'not_attempted' | 'captured' | 'unavailable' | 'failed';
|
|
193
|
-
page1OrganicCount: number;
|
|
194
|
-
page2OrganicCount: number;
|
|
195
|
-
failureCode?: 'missing_next' | 'invalid_next' | 'empty_page' | 'captcha' | 'timeout' | 'navigation_error' | 'unsupported_driver';
|
|
196
|
-
}
|
|
197
158
|
/** Material already observed before PAA expansion completes. It is checkpointed
|
|
198
159
|
* so a worker interruption cannot discard the SERP that led to the PAA graph. */
|
|
199
160
|
interface PaaProgressMaterial {
|
|
@@ -218,6 +179,119 @@ interface PaaProgressSnapshot {
|
|
|
218
179
|
material?: PaaProgressMaterial;
|
|
219
180
|
interactions?: PaaInteractionObservation[];
|
|
220
181
|
}
|
|
182
|
+
interface PaaSource extends SerpOutboundLink {
|
|
183
|
+
title: string | null;
|
|
184
|
+
site: string | null;
|
|
185
|
+
}
|
|
186
|
+
interface PAANode {
|
|
187
|
+
questionId: string;
|
|
188
|
+
googleLinkId: string | null;
|
|
189
|
+
googleEvidenceId: string | null;
|
|
190
|
+
question: string;
|
|
191
|
+
answer: string | null;
|
|
192
|
+
sourceTitle: string | null;
|
|
193
|
+
sourceSite: string | null;
|
|
194
|
+
sourceCite: string | null;
|
|
195
|
+
sources: PaaSource[];
|
|
196
|
+
depth: number;
|
|
197
|
+
parentQuestion: string | null;
|
|
198
|
+
children: PAANode[];
|
|
199
|
+
}
|
|
200
|
+
interface FlatRow {
|
|
201
|
+
seed_query: string;
|
|
202
|
+
question_id: string;
|
|
203
|
+
google_link_id: string;
|
|
204
|
+
google_evidence_id: string;
|
|
205
|
+
question: string;
|
|
206
|
+
answer: string;
|
|
207
|
+
source_title: string;
|
|
208
|
+
source_site: string;
|
|
209
|
+
source_cite: string;
|
|
210
|
+
sources: PaaSource[];
|
|
211
|
+
depth: number;
|
|
212
|
+
parent_question: string;
|
|
213
|
+
extracted_at: string;
|
|
214
|
+
}
|
|
215
|
+
type HarvestOptions = z.infer<typeof HarvestOptionsSchema>;
|
|
216
|
+
|
|
217
|
+
type CanonicalLocationName = string;
|
|
218
|
+
type SerpDevice = 'desktop' | 'mobile';
|
|
219
|
+
type ProxyMode = 'location' | 'configured' | 'none';
|
|
220
|
+
interface \u004b\u0065\u0072\u006e\u0065\u006cLocationProxyTarget {
|
|
221
|
+
canonicalLocation: CanonicalLocationName;
|
|
222
|
+
level: 'zip' | 'city' | 'state';
|
|
223
|
+
country: 'US';
|
|
224
|
+
state: string;
|
|
225
|
+
city: string;
|
|
226
|
+
zip?: string;
|
|
227
|
+
cityCandidates: string[];
|
|
228
|
+
proxyName: string;
|
|
229
|
+
config: {
|
|
230
|
+
country: 'US';
|
|
231
|
+
state: string;
|
|
232
|
+
city?: string;
|
|
233
|
+
zip?: string;
|
|
234
|
+
};
|
|
235
|
+
}
|
|
236
|
+
type \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionSource = 'disabled' | 'location_reused' | 'location_created' | 'isp_created' | 'backup_custom' | 'configured_fallback' | 'unavailable';
|
|
237
|
+
interface \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionDebug {
|
|
238
|
+
source: \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionSource;
|
|
239
|
+
proxyType: string;
|
|
240
|
+
proxyMode: ProxyMode;
|
|
241
|
+
proxyIdPresent: boolean;
|
|
242
|
+
proxyIdSuffix: string | null;
|
|
243
|
+
target: \u004b\u0065\u0072\u006e\u0065\u006cLocationProxyTarget | null;
|
|
244
|
+
error: string | null;
|
|
245
|
+
disposable?: boolean;
|
|
246
|
+
}
|
|
247
|
+
interface HarvestStats {
|
|
248
|
+
seed: string;
|
|
249
|
+
totalQuestions: number;
|
|
250
|
+
maxDepthReached: number;
|
|
251
|
+
durationMs: number;
|
|
252
|
+
errorCount: number;
|
|
253
|
+
}
|
|
254
|
+
type HarvestCompletionStatus = 'paa_found' | 'no_paa' | 'serp_only';
|
|
255
|
+
/**
|
|
256
|
+
* A completed browser attempt can still be unsuitable for programmatic use.
|
|
257
|
+
* This is deliberately separate from completionStatus so a valid no-PAA
|
|
258
|
+
* response with organic results is not confused with an unreadable SERP.
|
|
259
|
+
*/
|
|
260
|
+
type HarvestResultQuality = 'complete' | 'partial' | 'degraded';
|
|
261
|
+
type HarvestDegradationReason = 'empty_primary_serp' | 'page_two_missing';
|
|
262
|
+
type HarvestDiagnosticWarningCode = 'short_videos_captcha_skipped' | 'paa_partial';
|
|
263
|
+
type HarvestDiagnosticWarningSurface = 'short_videos' | 'paa';
|
|
264
|
+
interface HarvestDiagnosticWarning {
|
|
265
|
+
code: HarvestDiagnosticWarningCode;
|
|
266
|
+
surface: HarvestDiagnosticWarningSurface;
|
|
267
|
+
message: string;
|
|
268
|
+
retryable: boolean;
|
|
269
|
+
}
|
|
270
|
+
interface HarvestCompleteness {
|
|
271
|
+
paaWithoutAnswer: number;
|
|
272
|
+
paaWithoutSource: number;
|
|
273
|
+
paaAnswersRecovered: number;
|
|
274
|
+
aioShareCaptured: boolean | null;
|
|
275
|
+
paaRequested?: number;
|
|
276
|
+
paaReturned?: number;
|
|
277
|
+
paaUnique?: number;
|
|
278
|
+
}
|
|
279
|
+
type HarvestInterruptionCode = 'browser_session_interrupted' | 'request_aborted' | 'timeout';
|
|
280
|
+
type HarvestInterruptionReasonSource = 'client_runtime' | 'extractor';
|
|
281
|
+
interface HarvestInterruptionDiagnostic {
|
|
282
|
+
code: HarvestInterruptionCode;
|
|
283
|
+
reasonSource: HarvestInterruptionReasonSource;
|
|
284
|
+
message: string | null;
|
|
285
|
+
}
|
|
286
|
+
/** Observed organic pagination outcome; page two never expands a separate PAA graph. */
|
|
287
|
+
interface HarvestPaginationDiagnostics {
|
|
288
|
+
requestedPages: 1 | 2;
|
|
289
|
+
capturedPages: 1 | 2;
|
|
290
|
+
page2Status: 'not_requested' | 'not_attempted' | 'captured' | 'unavailable' | 'failed';
|
|
291
|
+
page1OrganicCount: number;
|
|
292
|
+
page2OrganicCount: number;
|
|
293
|
+
failureCode?: 'missing_next' | 'invalid_next' | 'empty_page' | 'captcha' | 'timeout' | 'navigation_error' | 'unsupported_driver';
|
|
294
|
+
}
|
|
221
295
|
interface HarvestDiagnostics {
|
|
222
296
|
phaseTimings?: {
|
|
223
297
|
providerConnectMs?: number;
|
|
@@ -265,7 +339,7 @@ interface BrowserNetworkLocationDebug {
|
|
|
265
339
|
timezone: string | null;
|
|
266
340
|
error: string | null;
|
|
267
341
|
}
|
|
268
|
-
interface
|
|
342
|
+
interface \u004b\u0065\u0072\u006e\u0065\u006cSessionDebug {
|
|
269
343
|
sessionId: string | null;
|
|
270
344
|
proxyMode: ProxyMode;
|
|
271
345
|
requestedProxyIdPresent: boolean;
|
|
@@ -277,7 +351,7 @@ interface KernelSessionDebug {
|
|
|
277
351
|
retrievedProxyIdMatchesRequested: boolean | null;
|
|
278
352
|
defaultProxyDisabled: boolean | null;
|
|
279
353
|
defaultProxyDisableError: string | null;
|
|
280
|
-
proxyResolution:
|
|
354
|
+
proxyResolution: \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionDebug | null;
|
|
281
355
|
timeoutSeconds: number | null;
|
|
282
356
|
stealth: boolean | null;
|
|
283
357
|
profilePresent: boolean | null;
|
|
@@ -307,7 +381,7 @@ interface SerpNavigationDebug {
|
|
|
307
381
|
redirected: boolean | null;
|
|
308
382
|
}
|
|
309
383
|
interface BrowserDebugSnapshot {
|
|
310
|
-
|
|
384
|
+
\u006b\u0065\u0072\u006e\u0065\u006c: \u004b\u0065\u0072\u006e\u0065\u006cSessionDebug | null;
|
|
311
385
|
context: BrowserContextDebug | null;
|
|
312
386
|
networkLocation: BrowserNetworkLocationDebug | null;
|
|
313
387
|
serpNavigation: SerpNavigationDebug | null;
|
|
@@ -358,39 +432,6 @@ interface SerpOutboundLink {
|
|
|
358
432
|
linkType: SerpLinkType;
|
|
359
433
|
resolutionStatus: SerpLinkResolutionStatus;
|
|
360
434
|
}
|
|
361
|
-
interface PaaSource extends SerpOutboundLink {
|
|
362
|
-
title: string | null;
|
|
363
|
-
site: string | null;
|
|
364
|
-
}
|
|
365
|
-
interface PAANode {
|
|
366
|
-
questionId: string;
|
|
367
|
-
googleLinkId: string | null;
|
|
368
|
-
googleEvidenceId: string | null;
|
|
369
|
-
question: string;
|
|
370
|
-
answer: string | null;
|
|
371
|
-
sourceTitle: string | null;
|
|
372
|
-
sourceSite: string | null;
|
|
373
|
-
sourceCite: string | null;
|
|
374
|
-
sources: PaaSource[];
|
|
375
|
-
depth: number;
|
|
376
|
-
parentQuestion: string | null;
|
|
377
|
-
children: PAANode[];
|
|
378
|
-
}
|
|
379
|
-
interface FlatRow {
|
|
380
|
-
seed_query: string;
|
|
381
|
-
question_id: string;
|
|
382
|
-
google_link_id: string;
|
|
383
|
-
google_evidence_id: string;
|
|
384
|
-
question: string;
|
|
385
|
-
answer: string;
|
|
386
|
-
source_title: string;
|
|
387
|
-
source_site: string;
|
|
388
|
-
source_cite: string;
|
|
389
|
-
sources: PaaSource[];
|
|
390
|
-
depth: number;
|
|
391
|
-
parent_question: string;
|
|
392
|
-
extracted_at: string;
|
|
393
|
-
}
|
|
394
435
|
interface VideoResult {
|
|
395
436
|
type: 'video' | 'short_video';
|
|
396
437
|
title: string;
|
|
@@ -495,9 +536,9 @@ interface HarvestResult {
|
|
|
495
536
|
organicResults: OrganicResult[];
|
|
496
537
|
localPack: LocalPackBusiness[];
|
|
497
538
|
entityIds: EntityIds;
|
|
539
|
+
fullSerpFeatures?: FullSerpFeatures;
|
|
498
540
|
stats: HarvestStats;
|
|
499
541
|
}
|
|
500
|
-
type HarvestOptions = z.infer<typeof HarvestOptionsSchema>;
|
|
501
542
|
|
|
502
543
|
declare function harvest(rawOptions: unknown): Promise<HarvestResult>;
|
|
503
544
|
|