mcp-scraper 0.97.1 → 0.98.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/CHANGELOG.md +5 -2442
  2. package/README.md +16 -412
  3. package/dist/bin/api-server.js +3 -2
  4. package/dist/bin/mcp-scraper-cli.js +33 -27
  5. package/dist/bin/mcp-scraper-core.js +9 -1
  6. package/dist/bin/mcp-scraper-install.js +17 -2
  7. package/dist/bin/mcp-stdio-server.js +9 -1
  8. package/dist/bin/paa-harvest.js +8 -1
  9. package/dist/index.cjs +5 -21830
  10. package/dist/index.d.ts +14 -607
  11. package/dist/index.js +5 -39
  12. package/dist/transport.d.ts +35 -0
  13. package/package.json +11 -51
  14. package/THIRD_PARTY_NOTICES.html +0 -203
  15. package/dist/actions-5X5CQAKO.js +0 -1
  16. package/dist/artifacts-4ZDNP2JW.js +0 -1
  17. package/dist/artifacts-N62OPIT5.js +0 -1
  18. package/dist/artifacts-PSADD5OC.js +0 -1
  19. package/dist/artifacts-RUR3OHAK.js +0 -1
  20. package/dist/blob-store-GIH26EDO.js +0 -1
  21. package/dist/bundle-GS657HXG.js +0 -1
  22. package/dist/chunk-24YDQ3EO.js +0 -5
  23. package/dist/chunk-2L7CHN5N.js +0 -4
  24. package/dist/chunk-3F7AFM53.js +0 -1
  25. package/dist/chunk-4AEW53RU.js +0 -954
  26. package/dist/chunk-4HZXATHB.js +0 -27
  27. package/dist/chunk-4YZ74SJ6.js +0 -1
  28. package/dist/chunk-4ZVO6GX4.js +0 -1
  29. package/dist/chunk-55ZANTHL.js +0 -3
  30. package/dist/chunk-5RQWFYXY.js +0 -930
  31. package/dist/chunk-5TQINRZN.js +0 -12
  32. package/dist/chunk-5UQAL7HV.js +0 -41
  33. package/dist/chunk-6D25Q5BR.js +0 -21728
  34. package/dist/chunk-6N5EC7ZO.js +0 -1
  35. package/dist/chunk-7POHNTVW.js +0 -1
  36. package/dist/chunk-7SR7MEU5.js +0 -62
  37. package/dist/chunk-ANQ4JJVI.js +0 -71
  38. package/dist/chunk-B3JRL45T.js +0 -1
  39. package/dist/chunk-B4ZKPLSP.js +0 -1
  40. package/dist/chunk-B5Y26E3A.js +0 -1
  41. package/dist/chunk-C3JWX6KM.js +0 -10
  42. package/dist/chunk-C7MGJUTU.js +0 -1
  43. package/dist/chunk-CNSVZTTT.js +0 -13
  44. package/dist/chunk-EYGA6LOU.js +0 -1
  45. package/dist/chunk-F3M7INSN.js +0 -41
  46. package/dist/chunk-FLAWC5PF.js +0 -5
  47. package/dist/chunk-FM7BX2LU.js +0 -159
  48. package/dist/chunk-HCK6PLM3.js +0 -3
  49. package/dist/chunk-HNNYXMSI.js +0 -1
  50. package/dist/chunk-IFPION5D.js +0 -121
  51. package/dist/chunk-IPQOBOFY.js +0 -1
  52. package/dist/chunk-J7XDXIB6.js +0 -163
  53. package/dist/chunk-JHDD6R3O.js +0 -8
  54. package/dist/chunk-JMZIDHYQ.js +0 -2
  55. package/dist/chunk-K3VDGJNK.js +0 -1
  56. package/dist/chunk-KBZY2OGP.js +0 -11
  57. package/dist/chunk-KJQXUZ4Y.js +0 -4
  58. package/dist/chunk-KXJCGV3H.js +0 -1
  59. package/dist/chunk-NALXKTUU.js +0 -1
  60. package/dist/chunk-NNOX3FYG.js +0 -198
  61. package/dist/chunk-NQFTETVC.js +0 -189
  62. package/dist/chunk-OILOHQNL.js +0 -2
  63. package/dist/chunk-OLYPLX5C.js +0 -1
  64. package/dist/chunk-P6CDR3RM.js +0 -1
  65. package/dist/chunk-PKQHHCO4.js +0 -20
  66. package/dist/chunk-QCBCOEH5.js +0 -1
  67. package/dist/chunk-QER2ELZP.js +0 -1
  68. package/dist/chunk-RDHGQ2WZ.js +0 -1
  69. package/dist/chunk-SDQW7IJN.js +0 -1
  70. package/dist/chunk-SEZETSB3.js +0 -1
  71. package/dist/chunk-SJNSTO3C.js +0 -42
  72. package/dist/chunk-TZYITYL4.js +0 -2
  73. package/dist/chunk-UIOSADPB.js +0 -1718
  74. package/dist/chunk-UNEJYZ4T.js +0 -102
  75. package/dist/chunk-UUJYICAJ.js +0 -1
  76. package/dist/chunk-VVSHZV5S.js +0 -73
  77. package/dist/chunk-WNUXMPVX.js +0 -1
  78. package/dist/chunk-XNTAHMPA.js +0 -218
  79. package/dist/chunk-ZDECILF6.js +0 -16
  80. package/dist/chunk-ZNPGM3XE.js +0 -4
  81. package/dist/compatibility-UPPKZ4KY.js +0 -1
  82. package/dist/contracts-GUXGFCVV.js +0 -1
  83. package/dist/core-schema-Y47YFKH2.js +0 -899
  84. package/dist/direct-completion-ZVPPEBDU.js +0 -1
  85. package/dist/dispatch-GRMMPKQD.js +0 -1
  86. package/dist/editorial-reading-room/assets/app.js +0 -433
  87. package/dist/editorial-reading-room/assets/index.html +0 -139
  88. package/dist/editorial-reading-room/assets/styles.css +0 -1091
  89. package/dist/index.d.cts +0 -607
  90. package/dist/jobs-NRZHGQ7I.js +0 -1
  91. package/dist/media-artifacts-67CJWI7O.js +0 -1
  92. package/dist/metering-YHZ46GV4.js +0 -1
  93. package/dist/migrations-3AKEC5QE.js +0 -1
  94. package/dist/queue-2VKW7SGH.js +0 -1
  95. package/dist/reconciliation-QC6SILLR.js +0 -1
  96. package/dist/report-artifact-offload-PQNFDNPL.js +0 -1
  97. package/dist/repository-CYXVMYAI.js +0 -1
  98. package/dist/repository-UKDA4IWP.js +0 -1
  99. package/dist/server-EYMJROPX.js +0 -5544
  100. package/dist/settlement-47EUTSGX.js +0 -1
  101. package/dist/vendor-errors-647LSH6H.js +0 -1
  102. package/dist/worker-AWJF7UYR.js +0 -1
  103. package/public/.well-known/xray-install.json +0 -183
  104. package/public/agent-install.md +0 -200
package/dist/index.d.cts DELETED
@@ -1,607 +0,0 @@
1
- import { z } from 'zod';
2
-
3
- declare const SERP_FEATURE_COMPLETENESS_STATES: readonly ["observed_present", "observed_absent", "incomplete", "not_requested", "unknown", "unsupported"];
4
- type SerpFeatureCompletenessState = typeof SERP_FEATURE_COMPLETENESS_STATES[number];
5
-
6
- type SerpLinkType = 'plain' | 'google_url_redirect' | 'google_goto_redirect';
7
- type SerpLinkResolutionStatus = 'not_needed' | 'resolved' | 'unresolved' | 'rejected';
8
- interface SerpOutboundLink {
9
- url: string;
10
- rawUrl: string;
11
- resolvedUrl: string | null;
12
- linkType: SerpLinkType;
13
- resolutionStatus: SerpLinkResolutionStatus;
14
- }
15
- interface VideoResult {
16
- type: 'video' | 'short_video';
17
- title: string;
18
- channel: string;
19
- platform: string;
20
- duration: string;
21
- url: string;
22
- }
23
- interface ForumResult {
24
- title: string;
25
- source: string;
26
- url: string;
27
- }
28
- interface WhatPeopleSayingCard {
29
- type: 'reddit' | 'facebook' | 'instagram' | 'tiktok' | 'youtube' | 'news' | 'unknown';
30
- title: string;
31
- url: string;
32
- source: string;
33
- platform: string;
34
- popularComment: string | null;
35
- engagement: string;
36
- date: string;
37
- duration: string | null;
38
- authorNote: string | null;
39
- }
40
- interface AICitation {
41
- text: string;
42
- href: string;
43
- rawUrl: string;
44
- resolvedUrl: string | null;
45
- linkType: SerpLinkType;
46
- resolutionStatus: SerpLinkResolutionStatus;
47
- }
48
- interface AIOverviewResult {
49
- detected: boolean;
50
- text: string | null;
51
- citations: AICitation[];
52
- expanded?: boolean;
53
- fullyExpanded?: boolean;
54
- sections?: string[];
55
- shareUrl?: string | null;
56
- }
57
- interface AIModeResult {
58
- detected: boolean;
59
- text: string | null;
60
- citations: AICitation[];
61
- }
62
- type GoogleSurface = 'web' | 'aim' | 'unknown';
63
- interface OrganicResult {
64
- position: number;
65
- title: string;
66
- url: string;
67
- rawUrl: string;
68
- resolvedUrl: string | null;
69
- linkType: SerpLinkType;
70
- resolutionStatus: SerpLinkResolutionStatus;
71
- domain: string;
72
- cite: string | null;
73
- snippet: string | null;
74
- isRedditStyle: boolean;
75
- inlineRating: {
76
- value: string;
77
- count: string;
78
- } | null;
79
- }
80
- interface LocalPackBusiness {
81
- position: number;
82
- name: string;
83
- cid: string | null;
84
- rating: string | null;
85
- reviewCount: string | null;
86
- metadata: string[];
87
- websiteUrl: string | null;
88
- directionsUrl: string | null;
89
- }
90
- interface EntityRecord {
91
- name: string;
92
- kgId: string | null;
93
- cid: string | null;
94
- gcid: string | null;
95
- }
96
- interface EntityIds {
97
- entities: EntityRecord[];
98
- kgIds: string[];
99
- cids: string[];
100
- gcids: string[];
101
- }
102
-
103
- interface SerpPreviewQuestion {
104
- question: string;
105
- answer: string | null;
106
- url: string | null;
107
- sourceTitle?: string | null;
108
- sourceName?: string | null;
109
- displayLink?: string | null;
110
- }
111
- interface SerpFeatureItem {
112
- title: string | null;
113
- url: string | null;
114
- description: string | null;
115
- position: number | null;
116
- }
117
- interface SerpFeatureGroup {
118
- name: string;
119
- attributes: Record<string, string | number | boolean | null>;
120
- items: SerpFeatureItem[];
121
- }
122
- interface FullSerpPageFeatures {
123
- page: 1 | 2;
124
- status: Record<string, SerpFeatureCompletenessState>;
125
- paaPreview: SerpPreviewQuestion[];
126
- additional: SerpFeatureGroup[];
127
- localPack: LocalPackBusiness[];
128
- forums: ForumResult[];
129
- videos: VideoResult[];
130
- whatPeopleSaying: WhatPeopleSayingCard[];
131
- aiOverview: AIOverviewResult;
132
- aiMode: AIModeResult;
133
- entityIds: EntityIds;
134
- }
135
- interface FullSerpFeatures {
136
- status: Record<string, SerpFeatureCompletenessState>;
137
- paaPreview: SerpPreviewQuestion[];
138
- additional: SerpFeatureGroup[];
139
- pages: FullSerpPageFeatures[];
140
- }
141
-
142
- declare const HarvestOptionsSchema: z.ZodObject<{
143
- query: z.ZodString;
144
- location: z.ZodOptional<z.ZodString>;
145
- gl: z.ZodDefault<z.ZodString>;
146
- hl: z.ZodDefault<z.ZodString>;
147
- device: z.ZodDefault<z.ZodEnum<{
148
- desktop: "desktop";
149
- mobile: "mobile";
150
- }>>;
151
- proxyMode: z.ZodDefault<z.ZodEnum<{
152
- location: "location";
153
- configured: "configured";
154
- none: "none";
155
- }>>;
156
- proxyZip: z.ZodOptional<z.ZodString>;
157
- keepDefaultProxy: z.ZodOptional<z.ZodBoolean>;
158
- requireUsEgress: z.ZodOptional<z.ZodBoolean>;
159
- maxAttempts: z.ZodOptional<z.ZodNumber>;
160
- debug: z.ZodDefault<z.ZodBoolean>;
161
- depth: z.ZodDefault<z.ZodNumber>;
162
- maxQuestions: z.ZodDefault<z.ZodNumber>;
163
- headless: z.ZodDefault<z.ZodBoolean>;
164
- profileDir: z.ZodOptional<z.ZodString>;
165
- proxy: z.ZodOptional<z.ZodString>;
166
- \u006b\u0065\u0072\u006e\u0065\u006cApiKey: z.ZodOptional<z.ZodString>;
167
- \u006b\u0065\u0072\u006e\u0065\u006cProxyId: z.ZodOptional<z.ZodString>;
168
- \u006b\u0065\u0072\u006e\u0065\u006cProfileName: z.ZodOptional<z.ZodString>;
169
- \u006b\u0065\u0072\u006e\u0065\u006cProfileSaveChanges: z.ZodOptional<z.ZodBoolean>;
170
- \u006b\u0065\u0072\u006e\u0065\u006cStealth: z.ZodOptional<z.ZodBoolean>;
171
- serpIdentity: z.ZodOptional<z.ZodString>;
172
- \u006b\u0065\u0072\u006e\u0065\u006cProxyResolution: z.ZodOptional<z.ZodUnknown>;
173
- outputDir: z.ZodDefault<z.ZodString>;
174
- format: z.ZodDefault<z.ZodEnum<{
175
- json: "json";
176
- csv: "csv";
177
- both: "both";
178
- }>>;
179
- serpOnly: z.ZodDefault<z.ZodBoolean>;
180
- questionsOnly: z.ZodDefault<z.ZodBoolean>;
181
- questionGrowthRecoveryRounds: z.ZodDefault<z.ZodNumber>;
182
- pages: z.ZodDefault<z.ZodNumber>;
183
- includeAllSerpFeatures: z.ZodDefault<z.ZodBoolean>;
184
- includeLocalPack: z.ZodDefault<z.ZodBoolean>;
185
- includeForums: z.ZodDefault<z.ZodBoolean>;
186
- includeVideos: z.ZodDefault<z.ZodBoolean>;
187
- includeAiOverview: z.ZodDefault<z.ZodBoolean>;
188
- includeWhatPeopleSaying: z.ZodDefault<z.ZodBoolean>;
189
- includeShortVideos: z.ZodDefault<z.ZodBoolean>;
190
- recency: z.ZodOptional<z.ZodEnum<{
191
- day: "day";
192
- week: "week";
193
- month: "month";
194
- year: "year";
195
- }>>;
196
- softDeadlineMs: z.ZodOptional<z.ZodNumber>;
197
- }, z.core.$strip>;
198
-
199
- type PaaDiscoveryStatus = 'in_progress' | 'target_reached' | 'frontier_exhausted' | 'no_paa_observed' | 'interrupted' | 'recovery_exhausted';
200
- type PaaTargetStatus = 'met' | 'not_met';
201
- type PaaMaterialStatus = 'complete' | 'partial';
202
- interface PaaRetryBreakdown {
203
- controlRetries: number;
204
- browserRetries: number;
205
- providerFallbacks: number;
206
- runtimeRecoveries: number;
207
- idempotentReplays: number;
208
- }
209
- interface PaaLifecycleDiagnostics {
210
- requestedQuestions: number;
211
- questionsFound: number;
212
- targetStatus: PaaTargetStatus;
213
- discoveryStatus: PaaDiscoveryStatus;
214
- materialStatus: PaaMaterialStatus;
215
- completionReason: string;
216
- automaticRetries: number;
217
- retryBreakdown: PaaRetryBreakdown;
218
- knownQuestions: number;
219
- processedQuestions: number;
220
- failedInteractions: number;
221
- noGrowthConfirmations: number;
222
- visibleControls?: number;
223
- eligibleControls?: number;
224
- dispatchedInteractions?: number;
225
- confirmedInteractions?: number;
226
- unconfirmedInteractions?: number;
227
- }
228
- type PaaInteractionOutcome = 'confirmed' | 'unconfirmed' | 'dispatch_failed';
229
- type PaaInteractionSignal = 'expanded' | 'answer' | 'source' | 'question_growth';
230
- /** One bounded, auditable interaction with a rendered PAA control. Question
231
- * growth is recorded separately from control-local answer/expanded signals so
232
- * a multi-control batch never pretends it can attribute shared DOM growth to a
233
- * specific click with more certainty than the browser exposed. */
234
- interface PaaInteractionObservation {
235
- id: string;
236
- questionId: string;
237
- question: string;
238
- batchNumber: number;
239
- controlAttempt: number;
240
- observedAt: string;
241
- dispatched: boolean;
242
- outcome: PaaInteractionOutcome;
243
- confirmationSignals: PaaInteractionSignal[];
244
- confirmationCheckpointMs: 700 | 1000 | 1400 | null;
245
- confirmedAtMs: number | null;
246
- dispatchDurationMs: number;
247
- totalDurationMs: number;
248
- beforeQuestionCount: number;
249
- afterQuestionCount: number;
250
- newQuestionCount: number;
251
- expandedBefore: boolean;
252
- expandedAfter: boolean;
253
- answerObserved: boolean;
254
- sourceCount: number;
255
- errorCode: 'click_failed' | 'click_ack_timeout' | 'control_missing' | 'confirmation_timeout' | null;
256
- }
257
- type PaaProgressPhase = 'serp_captured' | 'discovered' | 'click_observed' | 'answer_captured' | 'fill_incomplete' | 'expansion_finished' | 'attempt_ending';
258
- /** Material already observed before PAA expansion completes. It is checkpointed
259
- * so a worker interruption cannot discard the SERP that led to the PAA graph. */
260
- interface PaaProgressMaterial {
261
- pagination?: HarvestPaginationDiagnostics;
262
- surface: GoogleSurface;
263
- aiOverview: AIOverviewResult;
264
- aiMode: AIModeResult;
265
- whatPeopleSaying: WhatPeopleSayingCard[];
266
- videos: VideoResult[];
267
- forums: ForumResult[];
268
- organicResults: OrganicResult[];
269
- localPack: LocalPackBusiness[];
270
- entityIds: EntityIds;
271
- }
272
- interface PaaProgressSnapshot {
273
- sequence: number;
274
- observedAt: string;
275
- clickedQuestions: string[];
276
- phase: PaaProgressPhase;
277
- records: FlatRow[];
278
- lifecycle?: PaaLifecycleDiagnostics;
279
- material?: PaaProgressMaterial;
280
- interactions?: PaaInteractionObservation[];
281
- }
282
- interface PaaSource extends SerpOutboundLink {
283
- title: string | null;
284
- site: string | null;
285
- /** Description displayed in the Google source card, when observed. */
286
- description?: string | null;
287
- }
288
- interface PAANode {
289
- questionId: string;
290
- googleLinkId: string | null;
291
- googleEvidenceId: string | null;
292
- question: string;
293
- answer: string | null;
294
- /** A remaining inner answer control could not be expanded within the capture budget. */
295
- answerExpansionIncomplete?: boolean;
296
- sourceTitle: string | null;
297
- sourceSite: string | null;
298
- sourceCite: string | null;
299
- sources: PaaSource[];
300
- depth: number;
301
- parentQuestion: string | null;
302
- children: PAANode[];
303
- }
304
- interface FlatRow {
305
- seed_query: string;
306
- question_id: string;
307
- google_link_id: string;
308
- google_evidence_id: string;
309
- question: string;
310
- answer: string;
311
- answerExpansionIncomplete?: boolean;
312
- source_title: string;
313
- source_site: string;
314
- source_cite: string;
315
- sources: PaaSource[];
316
- depth: number;
317
- parent_question: string;
318
- extracted_at: string;
319
- }
320
- type HarvestOptions = z.infer<typeof HarvestOptionsSchema>;
321
-
322
- type CanonicalLocationName = string;
323
- type SerpDevice = 'desktop' | 'mobile';
324
- type ProxyMode = 'location' | 'configured' | 'none';
325
-
326
- interface \u004b\u0065\u0072\u006e\u0065\u006cLocationProxyTarget {
327
- canonicalLocation: CanonicalLocationName;
328
- level: 'zip' | 'city' | 'state';
329
- country: 'US';
330
- state: string;
331
- city: string;
332
- zip?: string;
333
- cityCandidates: string[];
334
- proxyName: string;
335
- config: {
336
- country: 'US';
337
- state: string;
338
- city?: string;
339
- zip?: string;
340
- };
341
- }
342
- type \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionSource = 'disabled' | 'location_reused' | 'location_created' | 'mobile_created' | 'isp_created' | 'backup_custom' | 'configured_fallback' | 'unavailable';
343
- interface \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionDebug {
344
- source: \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionSource;
345
- proxyType: string;
346
- proxyMode: ProxyMode;
347
- proxyIdPresent: boolean;
348
- proxyIdSuffix: string | null;
349
- target: \u004b\u0065\u0072\u006e\u0065\u006cLocationProxyTarget | null;
350
- error: string | null;
351
- disposable?: boolean;
352
- }
353
-
354
- interface BrowserNetworkLocationDebug {
355
- source: 'ipapi.co' | 'ipwho.is' | 'api64.ipify.org';
356
- ip: string | null;
357
- city: string | null;
358
- region: string | null;
359
- country: string | null;
360
- org: string | null;
361
- timezone: string | null;
362
- error: string | null;
363
- }
364
- interface \u004b\u0065\u0072\u006e\u0065\u006cSessionDebug {
365
- sessionId: string | null;
366
- proxyMode: ProxyMode;
367
- requestedProxyIdPresent: boolean;
368
- requestedProxyIdSuffix: string | null;
369
- createdProxyIdPresent: boolean | null;
370
- createdProxyIdSuffix: string | null;
371
- retrievedProxyIdPresent: boolean | null;
372
- retrievedProxyIdSuffix: string | null;
373
- retrievedProxyIdMatchesRequested: boolean | null;
374
- defaultProxyDisabled: boolean | null;
375
- defaultProxyDisableError: string | null;
376
- proxyResolution: \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionDebug | null;
377
- timeoutSeconds: number | null;
378
- stealth: boolean | null;
379
- profilePresent: boolean | null;
380
- poolPresent: boolean | null;
381
- retrieveError: string | null;
382
- }
383
- interface BrowserContextDebug {
384
- viewport: {
385
- width: number;
386
- height: number;
387
- } | null;
388
- locale: string | null;
389
- device: SerpDevice;
390
- requestedDevice?: SerpDevice;
391
- userAgent: string | null;
392
- deviceScaleFactor: number | null;
393
- isMobile: boolean;
394
- hasTouch: boolean;
395
- }
396
- interface SerpNavigationDebug {
397
- requestedUrl: string;
398
- finalUrl: string | null;
399
- title: string | null;
400
- bodySnippet: string | null;
401
- hasPaa: boolean | null;
402
- captchaDetected: boolean | null;
403
- googleSorryUrl: boolean | null;
404
- redirected: boolean | null;
405
- }
406
- interface BrowserDebugSnapshot {
407
- \u006b\u0065\u0072\u006e\u0065\u006c: \u004b\u0065\u0072\u006e\u0065\u006cSessionDebug | null;
408
- context: BrowserContextDebug | null;
409
- networkLocation: BrowserNetworkLocationDebug | null;
410
- serpNavigation: SerpNavigationDebug | null;
411
- }
412
-
413
- interface SerpLocationCandidate {
414
- city: string;
415
- regionCode: string;
416
- count: number;
417
- examples: string[];
418
- }
419
- interface SerpLocationEvidence {
420
- status: 'not_requested' | 'matched' | 'mismatch' | 'unknown';
421
- expected: {
422
- city: string;
423
- regionCode: string | null;
424
- canonicalLocation: string;
425
- } | null;
426
- candidates: SerpLocationCandidate[];
427
- }
428
-
429
- interface HarvestStats {
430
- seed: string;
431
- totalQuestions: number;
432
- maxDepthReached: number;
433
- durationMs: number;
434
- errorCount: number;
435
- }
436
- type HarvestCompletionStatus = 'paa_found' | 'no_paa' | 'serp_only';
437
- /**
438
- * A completed browser attempt can still be unsuitable for programmatic use.
439
- * This is deliberately separate from completionStatus so a valid no-PAA
440
- * response with organic results is not confused with an unreadable SERP.
441
- */
442
- type HarvestResultQuality = 'complete' | 'partial' | 'degraded';
443
- type HarvestDegradationReason = 'empty_primary_serp' | 'page_two_missing';
444
- type HarvestDiagnosticWarningCode = 'short_videos_captcha_skipped' | 'paa_partial';
445
- type HarvestDiagnosticWarningSurface = 'short_videos' | 'paa';
446
- interface HarvestDiagnosticWarning {
447
- code: HarvestDiagnosticWarningCode;
448
- surface: HarvestDiagnosticWarningSurface;
449
- message: string;
450
- retryable: boolean;
451
- }
452
- interface HarvestCompleteness {
453
- paaWithoutAnswer: number;
454
- paaWithoutSource: number;
455
- paaIncompleteAnswers?: number;
456
- paaAnswersRecovered: number;
457
- aioShareCaptured: boolean | null;
458
- paaRequested?: number;
459
- paaReturned?: number;
460
- paaUnique?: number;
461
- }
462
- type HarvestInterruptionCode = 'browser_session_interrupted' | 'request_aborted' | 'timeout';
463
- type HarvestInterruptionReasonSource = 'client_runtime' | 'extractor';
464
- interface HarvestInterruptionDiagnostic {
465
- code: HarvestInterruptionCode;
466
- reasonSource: HarvestInterruptionReasonSource;
467
- message: string | null;
468
- }
469
- /** Observed organic pagination outcome; page two never expands a separate PAA graph. */
470
- interface HarvestPaginationDiagnostics {
471
- requestedPages: 1 | 2;
472
- capturedPages: 1 | 2;
473
- page2Status: 'not_requested' | 'not_attempted' | 'captured' | 'unavailable' | 'failed';
474
- page1OrganicCount: number;
475
- page2OrganicCount: number;
476
- failureCode?: 'missing_next' | 'invalid_next' | 'empty_page' | 'captcha' | 'timeout' | 'navigation_error' | 'unsupported_driver';
477
- }
478
- interface HarvestDiagnostics {
479
- phaseTimings?: {
480
- providerConnectMs?: number;
481
- navigationMs?: number;
482
- serpParseMs?: number;
483
- paaExpansionMs?: number;
484
- linkResolutionMs?: number;
485
- totalServerMs: number;
486
- };
487
- pagination?: HarvestPaginationDiagnostics;
488
- completionStatus: HarvestCompletionStatus;
489
- /** Set only when the extractor directly observed a readable SERP without a PAA block. */
490
- noPaaObserved?: boolean;
491
- problem: null;
492
- resultQuality?: HarvestResultQuality;
493
- degradedResult?: boolean;
494
- degradationReasons?: HarvestDegradationReason[];
495
- retryRecommended?: boolean;
496
- interruption?: HarvestInterruptionDiagnostic;
497
- warnings?: HarvestDiagnosticWarning[];
498
- links?: HarvestLinkDiagnostics;
499
- completeness?: HarvestCompleteness;
500
- paaLifecycle?: PaaLifecycleDiagnostics;
501
- debug?: HarvestDebugSnapshot;
502
- }
503
- interface HarvestLinkDiagnostics {
504
- linkCandidatesObserved: number;
505
- plainLinksObserved: number;
506
- googleUrlRedirectsObserved: number;
507
- googleGotoRedirectsObserved: number;
508
- redirectsResolved: number;
509
- redirectsUnresolved: number;
510
- redirectsRejected: number;
511
- resolverNonGoogleRequestsObserved: number;
512
- paaSourcesPreserved: number;
513
- aiOverviewLabelValidated: boolean;
514
- }
515
- interface HarvestDebugSnapshot {
516
- enabled: true;
517
- request: {
518
- query: string;
519
- locationInput: string | null;
520
- canonicalLocation: string | null;
521
- locationResolved: boolean | null;
522
- locationMatchSource: string | null;
523
- locationAmbiguousCandidates: number | null;
524
- uule: string | null;
525
- gl: string;
526
- hl: string;
527
- device: SerpDevice;
528
- proxyMode: ProxyMode;
529
- proxyZip: string | null;
530
- serpIdentity: string | null;
531
- serpOnly: boolean;
532
- pages: number;
533
- };
534
- browser: BrowserDebugSnapshot;
535
- locationEvidence?: SerpLocationEvidence;
536
- }
537
- interface HarvestResult {
538
- seed: string;
539
- location: string | null;
540
- extractedAt: string;
541
- diagnostics: HarvestDiagnostics;
542
- totalQuestions: number;
543
- surface: GoogleSurface;
544
- aiOverview: AIOverviewResult;
545
- aiMode: AIModeResult;
546
- whatPeopleSaying: WhatPeopleSayingCard[];
547
- tree: PAANode[];
548
- flat: FlatRow[];
549
- videos: VideoResult[];
550
- forums: ForumResult[];
551
- organicResults: OrganicResult[];
552
- localPack: LocalPackBusiness[];
553
- entityIds: EntityIds;
554
- fullSerpFeatures?: FullSerpFeatures;
555
- stats: HarvestStats;
556
- }
557
-
558
- declare function harvest(rawOptions: unknown): Promise<HarvestResult>;
559
-
560
- interface EpisodeBrief {
561
- episodeNumber: number;
562
- episodeCount: number;
563
- slug: string;
564
- sectionTitle: string;
565
- storyMoment: string;
566
- characterName: string;
567
- clip1: string;
568
- clip2: string;
569
- voiceover: string;
570
- audioMood: string;
571
- depositionVisualTell: string;
572
- continuityNote: string;
573
- }
574
-
575
- interface ClipPairOptions {
576
- resolution?: '480p' | '720p' | '1080p';
577
- aspectRatio?: '16:9' | '9:16' | '1:1';
578
- clipDurationSeconds?: number;
579
- generateAudio?: boolean;
580
- seed?: number;
581
- outputDir?: string;
582
- }
583
- interface ClipPairResult {
584
- clip1Url: string;
585
- clip2Url: string;
586
- finalVideoPath: string;
587
- seed: number;
588
- promptClip1: string;
589
- promptClip2: string;
590
- voiceover: string;
591
- audioMood: string;
592
- }
593
- declare class VideoGenerator {
594
- constructor(apiKey?: string);
595
- generateClipPair(question: string, answer: string, opts?: ClipPairOptions): Promise<ClipPairResult>;
596
- generateEpisode(brief: EpisodeBrief, opts?: ClipPairOptions): Promise<ClipPairResult>;
597
- }
598
-
599
- interface ClipPromptPair {
600
- clip1: string;
601
- clip2: string;
602
- voiceover: string;
603
- audioMood: string;
604
- }
605
- declare function buildClipPrompts(question: string, answer: string): Promise<ClipPromptPair>;
606
-
607
- export { type AICitation, type AIModeResult, type AIOverviewResult, type ClipPairOptions, type ClipPairResult, type FlatRow, type GoogleSurface, type HarvestCompletionStatus, type HarvestDiagnostics, type HarvestOptions, type HarvestResult, type HarvestStats, type PAANode, type PaaDiscoveryStatus, type PaaLifecycleDiagnostics, type PaaMaterialStatus, type PaaProgressMaterial, type PaaProgressSnapshot, type PaaTargetStatus, VideoGenerator, type WhatPeopleSayingCard, buildClipPrompts, harvest };
@@ -1 +0,0 @@
1
- import{a,b,c,d,e,f,g}from"./chunk-FLAWC5PF.js";import"./chunk-C7MGJUTU.js";import"./chunk-WNUXMPVX.js";export{d as createJobWithId,e as createJobWithReceipt,b as deserialize,c as getJob,g as markJobBillingSettled,f as recordJobDispatchRequested,a as rowToRawJob};
@@ -1 +0,0 @@
1
- import{a,b,c,d,e,f}from"./chunk-HCK6PLM3.js";import"./chunk-JMZIDHYQ.js";import"./chunk-SEZETSB3.js";import"./chunk-SDQW7IJN.js";import"./chunk-OLYPLX5C.js";import"./chunk-B3JRL45T.js";import"./chunk-OILOHQNL.js";export{a as PAGE_MEDIA_ARTIFACT_PREFIX,b as PAGE_MEDIA_ARTIFACT_TTL_MS,f as packagePageMedia,d as pageMediaArtifactOwnerId,c as pageMediaArtifactPolicy,e as readOwnedPageMediaArtifactBuffer};
@@ -1 +0,0 @@
1
- import{a,b,c,d,e,f,g}from"./chunk-K3VDGJNK.js";import"./chunk-7SR7MEU5.js";import"./chunk-ANQ4JJVI.js";import"./chunk-PKQHHCO4.js";import"./chunk-4ZVO6GX4.js";import"./chunk-WNUXMPVX.js";import"./chunk-HNNYXMSI.js";export{g as finishExtractUrlCostRun,b as finishMeteredProviderAction,d as linkCurrentCostDebit,f as startExtractUrlCostRun,a as startMeteredProviderAction,c as withLocalCostPhase,e as withLogicalCostRun};
@@ -1 +0,0 @@
1
- import{a}from"./chunk-UIOSADPB.js";import"./chunk-NNOX3FYG.js";import"./chunk-NALXKTUU.js";export{a as migrateAnalytics};
@@ -1 +0,0 @@
1
- import{t as d,u,v as l,w as f,x as v,xb as S,zb as h}from"./chunk-5RQWFYXY.js";import"./chunk-NNOX3FYG.js";import"./chunk-FM7BX2LU.js";import"./chunk-NALXKTUU.js";import"./chunk-XNTAHMPA.js";import"./chunk-B5Y26E3A.js";import"./chunk-4YZ74SJ6.js";import"./chunk-JHDD6R3O.js";import"./chunk-VVSHZV5S.js";import"./chunk-4ZVO6GX4.js";import"./chunk-WNUXMPVX.js";import"./chunk-EYGA6LOU.js";var g=12;function b(e){return e&&typeof e=="object"&&"code"in e&&typeof e.code=="string"?e.code:e instanceof Error?e.name:"unknown_error"}function k(e){return Math.min(21600,30*2**Math.max(0,e-1))}async function x(e={}){let w=e.stripeClient??S,E=Math.max(1,Math.min(100,e.limit??100)),r=500,i=0,s=0,c=0,m=0;for(;i<r;){let a=await d(Math.min(E,r-i),e.leaseSeconds??300);if(a.length===0)break;i+=a.length;for(let t of a)try{let n=await w.events.retrieve(t.eventId);await h(n),await u(t.eventId,t.token)&&(s+=1)}catch(n){let p=n instanceof Error?n.message:String(n),o=b(n);t.attemptCount>=g||o==="resource_missing"?await f(t.eventId,t.token,p,o)&&(m+=1):await l(t.eventId,t.token,p,o,k(t.attemptCount))&&(c+=1)}}return{claimed:i,processed:s,retried:c,deadLettered:m,capped:i>=r,health:await v()}}export{x as drainStripeEventQueue};
@@ -1 +0,0 @@
1
- import{j as b,k as c}from"./chunk-P6CDR3RM.js";import{g as a}from"./chunk-UUJYICAJ.js";import"./chunk-5TQINRZN.js";import"./chunk-XNTAHMPA.js";import"./chunk-F3M7INSN.js";import"./chunk-JMZIDHYQ.js";import"./chunk-NQFTETVC.js";import"./chunk-KXJCGV3H.js";import"./chunk-C3JWX6KM.js";import"./chunk-QER2ELZP.js";import"./chunk-4YZ74SJ6.js";import"./chunk-JHDD6R3O.js";import"./chunk-CNSVZTTT.js";import"./chunk-SEZETSB3.js";import"./chunk-SDQW7IJN.js";import"./chunk-K3VDGJNK.js";import"./chunk-7SR7MEU5.js";import"./chunk-ANQ4JJVI.js";import"./chunk-PKQHHCO4.js";import"./chunk-4ZVO6GX4.js";import"./chunk-C7MGJUTU.js";import"./chunk-WNUXMPVX.js";import"./chunk-EYGA6LOU.js";import"./chunk-OLYPLX5C.js";import"./chunk-B3JRL45T.js";import"./chunk-OILOHQNL.js";import"./chunk-HNNYXMSI.js";export{c as reconcileSiteExtractSettlements,a as redispatchFundedSiteExtractJobs,b as refinalizeExtractJob};
@@ -1 +0,0 @@
1
- import{a,b,c,d,e,f,g,h,i,j,k,l,m,n,o}from"./chunk-TZYITYL4.js";import"./chunk-3F7AFM53.js";import"./chunk-RDHGQ2WZ.js";import"./chunk-SEZETSB3.js";import"./chunk-SDQW7IJN.js";import"./chunk-FLAWC5PF.js";import"./chunk-C7MGJUTU.js";import"./chunk-WNUXMPVX.js";import"./chunk-OILOHQNL.js";export{h as ARTIFACT_OFFLOAD_ENABLED,c as EXTRACT_URL_JOB_ARTIFACT_PREFIX,f as MCP_RESULT_ARTIFACT_PREFIX,d as PREVIEW_CHARS,e as REPORT_ARTIFACT_POLICY,b as REPORT_BLOB_PREFIX,a as REPORT_BLOB_TTL_MS,k as artifactOwnerId,n as artifactResourceLink,l as extractUrlJobResultArtifact,g as mcpResultArtifactPolicy,j as offloadLocalExtractUrlResult,i as offloadReport,m as readArtifactWindow,o as summaryEnvelope};
@@ -1 +0,0 @@
1
- import{a,b,c,d,e,f,g,h,i,j,k,l,m,n}from"./chunk-UNEJYZ4T.js";import"./chunk-WNUXMPVX.js";import"./chunk-KJQXUZ4Y.js";export{b as HOSTED_CENSUS_DATASET_PREFIX,a as HOSTED_ZIP_DATASET_KIND,c as MIN_NATIONWIDE_ZIP_COUNT,d as REQUIRED_LOCATION_STATE_CODES,g as ensureHostedLocationDataSchema,h as getActiveHostedDataset,i as getActiveHostedLocationDataset,k as getHostedZipGroups,m as importHostedCensusPlacesCsv,l as importHostedZipGroupsCsv,j as listActiveHostedLocationDatasets,e as parseHostedCensusPlacesCsv,f as parseHostedZipGroupsCsv,n as queryHostedLocationMarkets};
@@ -1 +0,0 @@
1
- import{a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u}from"./chunk-ANQ4JJVI.js";import"./chunk-PKQHHCO4.js";import"./chunk-4ZVO6GX4.js";import"./chunk-WNUXMPVX.js";import"./chunk-HNNYXMSI.js";export{a as claimExecution,d as claimOperationRecovery,l as createAdmittedOperation,m as finishOperationRun,p as hasOperationExecution,q as hasUnknownOperationGroup,j as isExecutionClaimCurrent,i as markExecutionUnknown,r as noActiveOperationGroupFence,u as noUnresolvedOperationGroupFence,n as operationExecutionFence,h as pauseWorkflowExecution,f as publishOperationRecovery,e as publishOperationResult,o as readOperationIntentResult,t as readStoredOperationIntent,g as releaseExecutionClaim,c as renewExecutionClaim,b as startExecutionEffects,s as startGuardedExecutionEffects,k as startOperationRun};