context.dev 2.18.0 → 2.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +220 -0
- data/README.md +1 -1
- data/lib/context_dev/client.rb +0 -4
- data/lib/context_dev/models/monitor_retrieve_run_params.rb +26 -0
- data/lib/context_dev/models/monitor_retrieve_run_response.rb +232 -0
- data/lib/context_dev/models/monitor_rotate_webhook_secret_params.rb +20 -0
- data/lib/context_dev/models/monitor_rotate_webhook_secret_response.rb +765 -0
- data/lib/context_dev/models/{web_web_scrape_sitemap_params.rb → web_map_urls_params.rb} +14 -14
- data/lib/context_dev/models/web_map_urls_response.rb +123 -0
- data/lib/context_dev/models/web_scrape_params.rb +1035 -0
- data/lib/context_dev/models/web_scrape_response.rb +974 -0
- data/lib/context_dev/models/web_screenshot_params.rb +15 -1
- data/lib/context_dev/models/web_screenshot_response.rb +3 -3
- data/lib/context_dev/models.rb +8 -22
- data/lib/context_dev/resources/brand.rb +0 -36
- data/lib/context_dev/resources/monitors.rb +47 -0
- data/lib/context_dev/resources/web.rb +118 -486
- data/lib/context_dev/version.rb +1 -1
- data/lib/context_dev.rb +8 -23
- data/rbi/context_dev/client.rbi +0 -3
- data/rbi/context_dev/models/monitor_retrieve_run_params.rbi +46 -0
- data/rbi/context_dev/models/monitor_retrieve_run_response.rbi +452 -0
- data/rbi/context_dev/models/monitor_rotate_webhook_secret_params.rbi +38 -0
- data/rbi/context_dev/models/monitor_rotate_webhook_secret_response.rbi +1367 -0
- data/rbi/context_dev/models/{web_web_scrape_sitemap_params.rbi → web_map_urls_params.rbi} +23 -45
- data/rbi/context_dev/models/web_map_urls_response.rbi +226 -0
- data/rbi/context_dev/models/web_scrape_params.rbi +2221 -0
- data/rbi/context_dev/models/web_scrape_response.rbi +1874 -0
- data/rbi/context_dev/models/web_screenshot_params.rbi +23 -0
- data/rbi/context_dev/models/web_screenshot_response.rbi +4 -4
- data/rbi/context_dev/models.rbi +9 -24
- data/rbi/context_dev/resources/brand.rbi +0 -35
- data/rbi/context_dev/resources/monitors.rbi +24 -0
- data/rbi/context_dev/resources/web.rbi +152 -654
- data/sig/context_dev/client.rbs +0 -2
- data/sig/context_dev/models/monitor_retrieve_run_params.rbs +28 -0
- data/sig/context_dev/models/monitor_retrieve_run_response.rbs +182 -0
- data/sig/context_dev/models/monitor_rotate_webhook_secret_params.rbs +23 -0
- data/sig/context_dev/models/monitor_rotate_webhook_secret_response.rbs +545 -0
- data/sig/context_dev/models/{web_web_scrape_sitemap_params.rbs → web_map_urls_params.rbs} +22 -22
- data/sig/context_dev/models/web_map_urls_response.rbs +125 -0
- data/sig/context_dev/models/web_scrape_params.rbs +925 -0
- data/sig/context_dev/models/web_scrape_response.rbs +782 -0
- data/sig/context_dev/models/web_screenshot_params.rbs +7 -0
- data/sig/context_dev/models.rbs +8 -22
- data/sig/context_dev/resources/brand.rbs +0 -9
- data/sig/context_dev/resources/monitors.rbs +11 -0
- data/sig/context_dev/resources/web.rbs +33 -128
- metadata +26 -71
- data/lib/context_dev/models/ai_extract_product_params.rb +0 -125
- data/lib/context_dev/models/ai_extract_product_response.rb +0 -402
- data/lib/context_dev/models/ai_extract_products_params.rb +0 -233
- data/lib/context_dev/models/ai_extract_products_response.rb +0 -370
- data/lib/context_dev/models/brand_retrieve_simplified_params.rb +0 -120
- data/lib/context_dev/models/brand_retrieve_simplified_response.rb +0 -441
- data/lib/context_dev/models/web_extract_fonts_params.rb +0 -114
- data/lib/context_dev/models/web_extract_fonts_response.rb +0 -291
- data/lib/context_dev/models/web_extract_params.rb +0 -419
- data/lib/context_dev/models/web_extract_response.rb +0 -287
- data/lib/context_dev/models/web_web_scrape_bytes_params.rb +0 -346
- data/lib/context_dev/models/web_web_scrape_bytes_response.rb +0 -135
- data/lib/context_dev/models/web_web_scrape_html_params.rb +0 -812
- data/lib/context_dev/models/web_web_scrape_html_response.rb +0 -628
- data/lib/context_dev/models/web_web_scrape_images_params.rb +0 -373
- data/lib/context_dev/models/web_web_scrape_images_response.rb +0 -409
- data/lib/context_dev/models/web_web_scrape_md_params.rb +0 -670
- data/lib/context_dev/models/web_web_scrape_md_response.rb +0 -567
- data/lib/context_dev/models/web_web_scrape_screenshot_params.rb +0 -471
- data/lib/context_dev/models/web_web_scrape_screenshot_response.rb +0 -169
- data/lib/context_dev/models/web_web_scrape_sitemap_response.rb +0 -144
- data/lib/context_dev/resources/ai.rb +0 -71
- data/rbi/context_dev/models/ai_extract_product_params.rbi +0 -253
- data/rbi/context_dev/models/ai_extract_product_response.rbi +0 -791
- data/rbi/context_dev/models/ai_extract_products_params.rbi +0 -490
- data/rbi/context_dev/models/ai_extract_products_response.rbi +0 -718
- data/rbi/context_dev/models/brand_retrieve_simplified_params.rbi +0 -256
- data/rbi/context_dev/models/brand_retrieve_simplified_response.rbi +0 -1040
- data/rbi/context_dev/models/web_extract_fonts_params.rbi +0 -223
- data/rbi/context_dev/models/web_extract_fonts_response.rbi +0 -563
- data/rbi/context_dev/models/web_extract_params.rbi +0 -781
- data/rbi/context_dev/models/web_extract_response.rbi +0 -512
- data/rbi/context_dev/models/web_web_scrape_bytes_params.rbi +0 -702
- data/rbi/context_dev/models/web_web_scrape_bytes_response.rbi +0 -238
- data/rbi/context_dev/models/web_web_scrape_html_params.rbi +0 -1690
- data/rbi/context_dev/models/web_web_scrape_html_response.rbi +0 -1287
- data/rbi/context_dev/models/web_web_scrape_images_params.rbi +0 -728
- data/rbi/context_dev/models/web_web_scrape_images_response.rbi +0 -910
- data/rbi/context_dev/models/web_web_scrape_md_params.rbi +0 -1052
- data/rbi/context_dev/models/web_web_scrape_md_response.rbi +0 -1088
- data/rbi/context_dev/models/web_web_scrape_screenshot_params.rbi +0 -1580
- data/rbi/context_dev/models/web_web_scrape_screenshot_response.rbi +0 -332
- data/rbi/context_dev/models/web_web_scrape_sitemap_response.rbi +0 -258
- data/rbi/context_dev/resources/ai.rbi +0 -63
- data/sig/context_dev/models/ai_extract_product_params.rbs +0 -106
- data/sig/context_dev/models/ai_extract_product_response.rbs +0 -314
- data/sig/context_dev/models/ai_extract_products_params.rbs +0 -202
- data/sig/context_dev/models/ai_extract_products_response.rbs +0 -293
- data/sig/context_dev/models/brand_retrieve_simplified_params.rbs +0 -104
- data/sig/context_dev/models/brand_retrieve_simplified_response.rbs +0 -413
- data/sig/context_dev/models/web_extract_fonts_params.rbs +0 -93
- data/sig/context_dev/models/web_extract_fonts_response.rbs +0 -230
- data/sig/context_dev/models/web_extract_params.rbs +0 -324
- data/sig/context_dev/models/web_extract_response.rbs +0 -239
- data/sig/context_dev/models/web_web_scrape_bytes_params.rbs +0 -531
- data/sig/context_dev/models/web_web_scrape_bytes_response.rbs +0 -108
- data/sig/context_dev/models/web_web_scrape_html_params.rbs +0 -878
- data/sig/context_dev/models/web_web_scrape_html_response.rbs +0 -525
- data/sig/context_dev/models/web_web_scrape_images_params.rbs +0 -281
- data/sig/context_dev/models/web_web_scrape_images_response.rbs +0 -371
- data/sig/context_dev/models/web_web_scrape_md_params.rbs +0 -758
- data/sig/context_dev/models/web_web_scrape_md_response.rbs +0 -465
- data/sig/context_dev/models/web_web_scrape_screenshot_params.rbs +0 -619
- data/sig/context_dev/models/web_web_scrape_screenshot_response.rbs +0 -127
- data/sig/context_dev/models/web_web_scrape_sitemap_response.rbs +0 -117
- data/sig/context_dev/resources/ai.rbs +0 -21
|
@@ -0,0 +1,2221 @@
|
|
|
1
|
+
# typed: strong
|
|
2
|
+
|
|
3
|
+
module ContextDev
|
|
4
|
+
module Models
|
|
5
|
+
class WebScrapeParams < ContextDev::Internal::Type::BaseModel
|
|
6
|
+
extend ContextDev::Internal::Type::RequestParameters::Converter
|
|
7
|
+
include ContextDev::Internal::Type::RequestParameters
|
|
8
|
+
|
|
9
|
+
OrHash =
|
|
10
|
+
T.type_alias do
|
|
11
|
+
T.any(ContextDev::WebScrapeParams, ContextDev::Internal::AnyHash)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# Outputs to return. Enable at least one; omitted formats are false.
|
|
15
|
+
sig { returns(ContextDev::WebScrapeParams::Formats) }
|
|
16
|
+
attr_reader :formats
|
|
17
|
+
|
|
18
|
+
sig { params(formats: ContextDev::WebScrapeParams::Formats::OrHash).void }
|
|
19
|
+
attr_writer :formats
|
|
20
|
+
|
|
21
|
+
# The URL to scrape.
|
|
22
|
+
sig { returns(String) }
|
|
23
|
+
attr_accessor :url
|
|
24
|
+
|
|
25
|
+
# Highlight options. Requires formats.highlights: true.
|
|
26
|
+
sig { returns(T.nilable(ContextDev::WebScrapeParams::HighlightsParams)) }
|
|
27
|
+
attr_reader :highlights_params
|
|
28
|
+
|
|
29
|
+
sig do
|
|
30
|
+
params(
|
|
31
|
+
highlights_params:
|
|
32
|
+
ContextDev::WebScrapeParams::HighlightsParams::OrHash
|
|
33
|
+
).void
|
|
34
|
+
end
|
|
35
|
+
attr_writer :highlights_params
|
|
36
|
+
|
|
37
|
+
# Image options. Requires formats.images: true.
|
|
38
|
+
sig { returns(T.nilable(ContextDev::WebScrapeParams::ImageParams)) }
|
|
39
|
+
attr_reader :image_params
|
|
40
|
+
|
|
41
|
+
sig do
|
|
42
|
+
params(
|
|
43
|
+
image_params: ContextDev::WebScrapeParams::ImageParams::OrHash
|
|
44
|
+
).void
|
|
45
|
+
end
|
|
46
|
+
attr_writer :image_params
|
|
47
|
+
|
|
48
|
+
# Required when formats.json is true.
|
|
49
|
+
sig { returns(T.nilable(ContextDev::WebScrapeParams::JsonParams)) }
|
|
50
|
+
attr_reader :json_params
|
|
51
|
+
|
|
52
|
+
sig do
|
|
53
|
+
params(
|
|
54
|
+
json_params: ContextDev::WebScrapeParams::JsonParams::OrHash
|
|
55
|
+
).void
|
|
56
|
+
end
|
|
57
|
+
attr_writer :json_params
|
|
58
|
+
|
|
59
|
+
# Markdown options. Requires formats.markdown: true.
|
|
60
|
+
sig { returns(T.nilable(ContextDev::WebScrapeParams::MarkdownParams)) }
|
|
61
|
+
attr_reader :markdown_params
|
|
62
|
+
|
|
63
|
+
sig do
|
|
64
|
+
params(
|
|
65
|
+
markdown_params: ContextDev::WebScrapeParams::MarkdownParams::OrHash
|
|
66
|
+
).void
|
|
67
|
+
end
|
|
68
|
+
attr_writer :markdown_params
|
|
69
|
+
|
|
70
|
+
# Maximum age of each cached output. Defaults to 1 day; 0 fetches fresh and
|
|
71
|
+
# updates the requested outputs. Compatible outputs are shared with the individual
|
|
72
|
+
# scrape endpoints. Image results with hosted files refresh after 23 hours; other
|
|
73
|
+
# outputs retain their own freshness.
|
|
74
|
+
sig { returns(T.nilable(Integer)) }
|
|
75
|
+
attr_reader :max_age_ms
|
|
76
|
+
|
|
77
|
+
sig { params(max_age_ms: Integer).void }
|
|
78
|
+
attr_writer :max_age_ms
|
|
79
|
+
|
|
80
|
+
# Required when formats.parse is true.
|
|
81
|
+
sig { returns(T.nilable(ContextDev::WebScrapeParams::ParseParams)) }
|
|
82
|
+
attr_reader :parse_params
|
|
83
|
+
|
|
84
|
+
sig do
|
|
85
|
+
params(
|
|
86
|
+
parse_params: ContextDev::WebScrapeParams::ParseParams::OrHash
|
|
87
|
+
).void
|
|
88
|
+
end
|
|
89
|
+
attr_writer :parse_params
|
|
90
|
+
|
|
91
|
+
# Product options. Requires formats.product: true.
|
|
92
|
+
sig { returns(T.nilable(ContextDev::WebScrapeParams::ProductParams)) }
|
|
93
|
+
attr_reader :product_params
|
|
94
|
+
|
|
95
|
+
sig do
|
|
96
|
+
params(
|
|
97
|
+
product_params: ContextDev::WebScrapeParams::ProductParams::OrHash
|
|
98
|
+
).void
|
|
99
|
+
end
|
|
100
|
+
attr_writer :product_params
|
|
101
|
+
|
|
102
|
+
# Screenshot options. Requires formats.screenshot: true.
|
|
103
|
+
sig { returns(T.nilable(ContextDev::WebScrapeParams::ScreenshotParams)) }
|
|
104
|
+
attr_reader :screenshot_params
|
|
105
|
+
|
|
106
|
+
sig do
|
|
107
|
+
params(
|
|
108
|
+
screenshot_params:
|
|
109
|
+
ContextDev::WebScrapeParams::ScreenshotParams::OrHash
|
|
110
|
+
).void
|
|
111
|
+
end
|
|
112
|
+
attr_writer :screenshot_params
|
|
113
|
+
|
|
114
|
+
# Shared browser and content settings. Content filters leave screenshots and
|
|
115
|
+
# original bytes unchanged.
|
|
116
|
+
sig { returns(T.nilable(ContextDev::WebScrapeParams::SharedParams)) }
|
|
117
|
+
attr_reader :shared_params
|
|
118
|
+
|
|
119
|
+
sig do
|
|
120
|
+
params(
|
|
121
|
+
shared_params: ContextDev::WebScrapeParams::SharedParams::OrHash
|
|
122
|
+
).void
|
|
123
|
+
end
|
|
124
|
+
attr_writer :shared_params
|
|
125
|
+
|
|
126
|
+
# Labels for tracking request usage. Not retained when zdr is enabled.
|
|
127
|
+
sig { returns(T.nilable(T::Array[String])) }
|
|
128
|
+
attr_reader :tags
|
|
129
|
+
|
|
130
|
+
sig { params(tags: T::Array[String]).void }
|
|
131
|
+
attr_writer :tags
|
|
132
|
+
|
|
133
|
+
# Total deadline, including navigation, actions, waiting, and all outputs.
|
|
134
|
+
# Defaults to 60000 milliseconds with behavior fail. Use return-partial to capture
|
|
135
|
+
# the current page state and return captured images if image processing cannot
|
|
136
|
+
# finish before the deadline; these responses set isPartial and are not cached.
|
|
137
|
+
# Every requested format must still be available. Fixed waits must fit before a
|
|
138
|
+
# response reserve of up to 5000 milliseconds (at most one quarter of the timeout)
|
|
139
|
+
# when using return-partial.
|
|
140
|
+
sig { returns(T.nilable(ContextDev::WebScrapeParams::TimeoutOpts)) }
|
|
141
|
+
attr_reader :timeout_opts
|
|
142
|
+
|
|
143
|
+
sig do
|
|
144
|
+
params(
|
|
145
|
+
timeout_opts: ContextDev::WebScrapeParams::TimeoutOpts::OrHash
|
|
146
|
+
).void
|
|
147
|
+
end
|
|
148
|
+
attr_writer :timeout_opts
|
|
149
|
+
|
|
150
|
+
# Zero data retention. Bypasses caches and uploads; excludes request/response
|
|
151
|
+
# content and tags from logs. Must be enabled for your organization. Not available
|
|
152
|
+
# with the highlights output.
|
|
153
|
+
sig { returns(T.nilable(ContextDev::WebScrapeParams::Zdr::OrSymbol)) }
|
|
154
|
+
attr_reader :zdr
|
|
155
|
+
|
|
156
|
+
sig { params(zdr: ContextDev::WebScrapeParams::Zdr::OrSymbol).void }
|
|
157
|
+
attr_writer :zdr
|
|
158
|
+
|
|
159
|
+
sig do
|
|
160
|
+
params(
|
|
161
|
+
formats: ContextDev::WebScrapeParams::Formats::OrHash,
|
|
162
|
+
url: String,
|
|
163
|
+
highlights_params:
|
|
164
|
+
ContextDev::WebScrapeParams::HighlightsParams::OrHash,
|
|
165
|
+
image_params: ContextDev::WebScrapeParams::ImageParams::OrHash,
|
|
166
|
+
json_params: ContextDev::WebScrapeParams::JsonParams::OrHash,
|
|
167
|
+
markdown_params: ContextDev::WebScrapeParams::MarkdownParams::OrHash,
|
|
168
|
+
max_age_ms: Integer,
|
|
169
|
+
parse_params: ContextDev::WebScrapeParams::ParseParams::OrHash,
|
|
170
|
+
product_params: ContextDev::WebScrapeParams::ProductParams::OrHash,
|
|
171
|
+
screenshot_params:
|
|
172
|
+
ContextDev::WebScrapeParams::ScreenshotParams::OrHash,
|
|
173
|
+
shared_params: ContextDev::WebScrapeParams::SharedParams::OrHash,
|
|
174
|
+
tags: T::Array[String],
|
|
175
|
+
timeout_opts: ContextDev::WebScrapeParams::TimeoutOpts::OrHash,
|
|
176
|
+
zdr: ContextDev::WebScrapeParams::Zdr::OrSymbol,
|
|
177
|
+
request_options: ContextDev::RequestOptions::OrHash
|
|
178
|
+
).returns(T.attached_class)
|
|
179
|
+
end
|
|
180
|
+
def self.new(
|
|
181
|
+
# Outputs to return. Enable at least one; omitted formats are false.
|
|
182
|
+
formats:,
|
|
183
|
+
# The URL to scrape.
|
|
184
|
+
url:,
|
|
185
|
+
# Highlight options. Requires formats.highlights: true.
|
|
186
|
+
highlights_params: nil,
|
|
187
|
+
# Image options. Requires formats.images: true.
|
|
188
|
+
image_params: nil,
|
|
189
|
+
# Required when formats.json is true.
|
|
190
|
+
json_params: nil,
|
|
191
|
+
# Markdown options. Requires formats.markdown: true.
|
|
192
|
+
markdown_params: nil,
|
|
193
|
+
# Maximum age of each cached output. Defaults to 1 day; 0 fetches fresh and
|
|
194
|
+
# updates the requested outputs. Compatible outputs are shared with the individual
|
|
195
|
+
# scrape endpoints. Image results with hosted files refresh after 23 hours; other
|
|
196
|
+
# outputs retain their own freshness.
|
|
197
|
+
max_age_ms: nil,
|
|
198
|
+
# Required when formats.parse is true.
|
|
199
|
+
parse_params: nil,
|
|
200
|
+
# Product options. Requires formats.product: true.
|
|
201
|
+
product_params: nil,
|
|
202
|
+
# Screenshot options. Requires formats.screenshot: true.
|
|
203
|
+
screenshot_params: nil,
|
|
204
|
+
# Shared browser and content settings. Content filters leave screenshots and
|
|
205
|
+
# original bytes unchanged.
|
|
206
|
+
shared_params: nil,
|
|
207
|
+
# Labels for tracking request usage. Not retained when zdr is enabled.
|
|
208
|
+
tags: nil,
|
|
209
|
+
# Total deadline, including navigation, actions, waiting, and all outputs.
|
|
210
|
+
# Defaults to 60000 milliseconds with behavior fail. Use return-partial to capture
|
|
211
|
+
# the current page state and return captured images if image processing cannot
|
|
212
|
+
# finish before the deadline; these responses set isPartial and are not cached.
|
|
213
|
+
# Every requested format must still be available. Fixed waits must fit before a
|
|
214
|
+
# response reserve of up to 5000 milliseconds (at most one quarter of the timeout)
|
|
215
|
+
# when using return-partial.
|
|
216
|
+
timeout_opts: nil,
|
|
217
|
+
# Zero data retention. Bypasses caches and uploads; excludes request/response
|
|
218
|
+
# content and tags from logs. Must be enabled for your organization. Not available
|
|
219
|
+
# with the highlights output.
|
|
220
|
+
zdr: nil,
|
|
221
|
+
request_options: {}
|
|
222
|
+
)
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
sig do
|
|
226
|
+
override.returns(
|
|
227
|
+
{
|
|
228
|
+
formats: ContextDev::WebScrapeParams::Formats,
|
|
229
|
+
url: String,
|
|
230
|
+
highlights_params: ContextDev::WebScrapeParams::HighlightsParams,
|
|
231
|
+
image_params: ContextDev::WebScrapeParams::ImageParams,
|
|
232
|
+
json_params: ContextDev::WebScrapeParams::JsonParams,
|
|
233
|
+
markdown_params: ContextDev::WebScrapeParams::MarkdownParams,
|
|
234
|
+
max_age_ms: Integer,
|
|
235
|
+
parse_params: ContextDev::WebScrapeParams::ParseParams,
|
|
236
|
+
product_params: ContextDev::WebScrapeParams::ProductParams,
|
|
237
|
+
screenshot_params: ContextDev::WebScrapeParams::ScreenshotParams,
|
|
238
|
+
shared_params: ContextDev::WebScrapeParams::SharedParams,
|
|
239
|
+
tags: T::Array[String],
|
|
240
|
+
timeout_opts: ContextDev::WebScrapeParams::TimeoutOpts,
|
|
241
|
+
zdr: ContextDev::WebScrapeParams::Zdr::OrSymbol,
|
|
242
|
+
request_options: ContextDev::RequestOptions
|
|
243
|
+
}
|
|
244
|
+
)
|
|
245
|
+
end
|
|
246
|
+
def to_hash
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
class Formats < ContextDev::Internal::Type::BaseModel
|
|
250
|
+
OrHash =
|
|
251
|
+
T.type_alias do
|
|
252
|
+
T.any(
|
|
253
|
+
ContextDev::WebScrapeParams::Formats,
|
|
254
|
+
ContextDev::Internal::AnyHash
|
|
255
|
+
)
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
# The original HTTP response body.
|
|
259
|
+
sig { returns(T.nilable(T::Boolean)) }
|
|
260
|
+
attr_reader :bytes
|
|
261
|
+
|
|
262
|
+
sig { params(bytes: T::Boolean).void }
|
|
263
|
+
attr_writer :bytes
|
|
264
|
+
|
|
265
|
+
# Plain-text passages from the page that are most relevant to
|
|
266
|
+
# highlightsParams.query, each prefixed with its section heading. Adds 3 credits.
|
|
267
|
+
# Not available with zdr enabled.
|
|
268
|
+
sig { returns(T.nilable(T::Boolean)) }
|
|
269
|
+
attr_reader :highlights
|
|
270
|
+
|
|
271
|
+
sig { params(highlights: T::Boolean).void }
|
|
272
|
+
attr_writer :highlights
|
|
273
|
+
|
|
274
|
+
# Rendered HTML.
|
|
275
|
+
sig { returns(T.nilable(T::Boolean)) }
|
|
276
|
+
attr_reader :html
|
|
277
|
+
|
|
278
|
+
sig { params(html: T::Boolean).void }
|
|
279
|
+
attr_writer :html
|
|
280
|
+
|
|
281
|
+
# Images found on the page.
|
|
282
|
+
sig { returns(T.nilable(T::Boolean)) }
|
|
283
|
+
attr_reader :images
|
|
284
|
+
|
|
285
|
+
sig { params(images: T::Boolean).void }
|
|
286
|
+
attr_writer :images
|
|
287
|
+
|
|
288
|
+
# Page data extracted by an LLM from the page Markdown into jsonParams.schema;
|
|
289
|
+
# values carried only in attributes or CSS classes need formats.parse instead.
|
|
290
|
+
# Adds four credits when the page has text to extract; when shared content filters
|
|
291
|
+
# leave no text the result is an empty object and only the base price applies.
|
|
292
|
+
sig { returns(T.nilable(T::Boolean)) }
|
|
293
|
+
attr_reader :json
|
|
294
|
+
|
|
295
|
+
sig { params(json: T::Boolean).void }
|
|
296
|
+
attr_writer :json
|
|
297
|
+
|
|
298
|
+
# Page content as Markdown.
|
|
299
|
+
sig { returns(T.nilable(T::Boolean)) }
|
|
300
|
+
attr_reader :markdown
|
|
301
|
+
|
|
302
|
+
sig { params(markdown: T::Boolean).void }
|
|
303
|
+
attr_writer :markdown
|
|
304
|
+
|
|
305
|
+
# Fields selected by parseParams.rules.
|
|
306
|
+
sig { returns(T.nilable(T::Boolean)) }
|
|
307
|
+
attr_reader :parse
|
|
308
|
+
|
|
309
|
+
sig { params(parse: T::Boolean).void }
|
|
310
|
+
attr_writer :parse
|
|
311
|
+
|
|
312
|
+
# Structured product data for product detail pages. Adds one credit.
|
|
313
|
+
sig { returns(T.nilable(T::Boolean)) }
|
|
314
|
+
attr_reader :product
|
|
315
|
+
|
|
316
|
+
sig { params(product: T::Boolean).void }
|
|
317
|
+
attr_writer :product
|
|
318
|
+
|
|
319
|
+
# An inline image of the page.
|
|
320
|
+
sig { returns(T.nilable(T::Boolean)) }
|
|
321
|
+
attr_reader :screenshot
|
|
322
|
+
|
|
323
|
+
sig { params(screenshot: T::Boolean).void }
|
|
324
|
+
attr_writer :screenshot
|
|
325
|
+
|
|
326
|
+
# Outputs to return. Enable at least one; omitted formats are false.
|
|
327
|
+
sig do
|
|
328
|
+
params(
|
|
329
|
+
bytes: T::Boolean,
|
|
330
|
+
highlights: T::Boolean,
|
|
331
|
+
html: T::Boolean,
|
|
332
|
+
images: T::Boolean,
|
|
333
|
+
json: T::Boolean,
|
|
334
|
+
markdown: T::Boolean,
|
|
335
|
+
parse: T::Boolean,
|
|
336
|
+
product: T::Boolean,
|
|
337
|
+
screenshot: T::Boolean
|
|
338
|
+
).returns(T.attached_class)
|
|
339
|
+
end
|
|
340
|
+
def self.new(
|
|
341
|
+
# The original HTTP response body.
|
|
342
|
+
bytes: nil,
|
|
343
|
+
# Plain-text passages from the page that are most relevant to
|
|
344
|
+
# highlightsParams.query, each prefixed with its section heading. Adds 3 credits.
|
|
345
|
+
# Not available with zdr enabled.
|
|
346
|
+
highlights: nil,
|
|
347
|
+
# Rendered HTML.
|
|
348
|
+
html: nil,
|
|
349
|
+
# Images found on the page.
|
|
350
|
+
images: nil,
|
|
351
|
+
# Page data extracted by an LLM from the page Markdown into jsonParams.schema;
|
|
352
|
+
# values carried only in attributes or CSS classes need formats.parse instead.
|
|
353
|
+
# Adds four credits when the page has text to extract; when shared content filters
|
|
354
|
+
# leave no text the result is an empty object and only the base price applies.
|
|
355
|
+
json: nil,
|
|
356
|
+
# Page content as Markdown.
|
|
357
|
+
markdown: nil,
|
|
358
|
+
# Fields selected by parseParams.rules.
|
|
359
|
+
parse: nil,
|
|
360
|
+
# Structured product data for product detail pages. Adds one credit.
|
|
361
|
+
product: nil,
|
|
362
|
+
# An inline image of the page.
|
|
363
|
+
screenshot: nil
|
|
364
|
+
)
|
|
365
|
+
end
|
|
366
|
+
|
|
367
|
+
sig do
|
|
368
|
+
override.returns(
|
|
369
|
+
{
|
|
370
|
+
bytes: T::Boolean,
|
|
371
|
+
highlights: T::Boolean,
|
|
372
|
+
html: T::Boolean,
|
|
373
|
+
images: T::Boolean,
|
|
374
|
+
json: T::Boolean,
|
|
375
|
+
markdown: T::Boolean,
|
|
376
|
+
parse: T::Boolean,
|
|
377
|
+
product: T::Boolean,
|
|
378
|
+
screenshot: T::Boolean
|
|
379
|
+
}
|
|
380
|
+
)
|
|
381
|
+
end
|
|
382
|
+
def to_hash
|
|
383
|
+
end
|
|
384
|
+
end
|
|
385
|
+
|
|
386
|
+
class HighlightsParams < ContextDev::Internal::Type::BaseModel
|
|
387
|
+
OrHash =
|
|
388
|
+
T.type_alias do
|
|
389
|
+
T.any(
|
|
390
|
+
ContextDev::WebScrapeParams::HighlightsParams,
|
|
391
|
+
ContextDev::Internal::AnyHash
|
|
392
|
+
)
|
|
393
|
+
end
|
|
394
|
+
|
|
395
|
+
# The question or topic to find passages for.
|
|
396
|
+
sig { returns(String) }
|
|
397
|
+
attr_accessor :query
|
|
398
|
+
|
|
399
|
+
# Maximum combined length of the returned passages, in characters.
|
|
400
|
+
sig { returns(T.nilable(Integer)) }
|
|
401
|
+
attr_reader :max_characters
|
|
402
|
+
|
|
403
|
+
sig { params(max_characters: Integer).void }
|
|
404
|
+
attr_writer :max_characters
|
|
405
|
+
|
|
406
|
+
# Highlight options. Requires formats.highlights: true.
|
|
407
|
+
sig do
|
|
408
|
+
params(query: String, max_characters: Integer).returns(
|
|
409
|
+
T.attached_class
|
|
410
|
+
)
|
|
411
|
+
end
|
|
412
|
+
def self.new(
|
|
413
|
+
# The question or topic to find passages for.
|
|
414
|
+
query:,
|
|
415
|
+
# Maximum combined length of the returned passages, in characters.
|
|
416
|
+
max_characters: nil
|
|
417
|
+
)
|
|
418
|
+
end
|
|
419
|
+
|
|
420
|
+
sig { override.returns({ query: String, max_characters: Integer }) }
|
|
421
|
+
def to_hash
|
|
422
|
+
end
|
|
423
|
+
end
|
|
424
|
+
|
|
425
|
+
class ImageParams < ContextDev::Internal::Type::BaseModel
|
|
426
|
+
OrHash =
|
|
427
|
+
T.type_alias do
|
|
428
|
+
T.any(
|
|
429
|
+
ContextDev::WebScrapeParams::ImageParams,
|
|
430
|
+
ContextDev::Internal::AnyHash
|
|
431
|
+
)
|
|
432
|
+
end
|
|
433
|
+
|
|
434
|
+
# For visual duplicates, keep the largest image.
|
|
435
|
+
sig do
|
|
436
|
+
returns(
|
|
437
|
+
T.nilable(
|
|
438
|
+
ContextDev::WebScrapeParams::ImageParams::Dedupe::OrSymbol
|
|
439
|
+
)
|
|
440
|
+
)
|
|
441
|
+
end
|
|
442
|
+
attr_reader :dedupe
|
|
443
|
+
|
|
444
|
+
sig do
|
|
445
|
+
params(
|
|
446
|
+
dedupe: ContextDev::WebScrapeParams::ImageParams::Dedupe::OrSymbol
|
|
447
|
+
).void
|
|
448
|
+
end
|
|
449
|
+
attr_writer :dedupe
|
|
450
|
+
|
|
451
|
+
# Add dimensions, a visual category, or a hosted file URL. Each image has a
|
|
452
|
+
# maximum processing time of 30000 milliseconds, bounded by the remaining request
|
|
453
|
+
# deadline.
|
|
454
|
+
sig do
|
|
455
|
+
returns(
|
|
456
|
+
T.nilable(
|
|
457
|
+
T::Array[
|
|
458
|
+
ContextDev::WebScrapeParams::ImageParams::Enrich::OrSymbol
|
|
459
|
+
]
|
|
460
|
+
)
|
|
461
|
+
)
|
|
462
|
+
end
|
|
463
|
+
attr_reader :enrich
|
|
464
|
+
|
|
465
|
+
sig do
|
|
466
|
+
params(
|
|
467
|
+
enrich:
|
|
468
|
+
T::Array[
|
|
469
|
+
ContextDev::WebScrapeParams::ImageParams::Enrich::OrSymbol
|
|
470
|
+
]
|
|
471
|
+
).void
|
|
472
|
+
end
|
|
473
|
+
attr_writer :enrich
|
|
474
|
+
|
|
475
|
+
# Image options. Requires formats.images: true.
|
|
476
|
+
sig do
|
|
477
|
+
params(
|
|
478
|
+
dedupe: ContextDev::WebScrapeParams::ImageParams::Dedupe::OrSymbol,
|
|
479
|
+
enrich:
|
|
480
|
+
T::Array[
|
|
481
|
+
ContextDev::WebScrapeParams::ImageParams::Enrich::OrSymbol
|
|
482
|
+
]
|
|
483
|
+
).returns(T.attached_class)
|
|
484
|
+
end
|
|
485
|
+
def self.new(
|
|
486
|
+
# For visual duplicates, keep the largest image.
|
|
487
|
+
dedupe: nil,
|
|
488
|
+
# Add dimensions, a visual category, or a hosted file URL. Each image has a
|
|
489
|
+
# maximum processing time of 30000 milliseconds, bounded by the remaining request
|
|
490
|
+
# deadline.
|
|
491
|
+
enrich: nil
|
|
492
|
+
)
|
|
493
|
+
end
|
|
494
|
+
|
|
495
|
+
sig do
|
|
496
|
+
override.returns(
|
|
497
|
+
{
|
|
498
|
+
dedupe:
|
|
499
|
+
ContextDev::WebScrapeParams::ImageParams::Dedupe::OrSymbol,
|
|
500
|
+
enrich:
|
|
501
|
+
T::Array[
|
|
502
|
+
ContextDev::WebScrapeParams::ImageParams::Enrich::OrSymbol
|
|
503
|
+
]
|
|
504
|
+
}
|
|
505
|
+
)
|
|
506
|
+
end
|
|
507
|
+
def to_hash
|
|
508
|
+
end
|
|
509
|
+
|
|
510
|
+
# For visual duplicates, keep the largest image.
|
|
511
|
+
module Dedupe
|
|
512
|
+
extend ContextDev::Internal::Type::Enum
|
|
513
|
+
|
|
514
|
+
TaggedSymbol =
|
|
515
|
+
T.type_alias do
|
|
516
|
+
T.all(Symbol, ContextDev::WebScrapeParams::ImageParams::Dedupe)
|
|
517
|
+
end
|
|
518
|
+
OrSymbol = T.type_alias { T.any(Symbol, String) }
|
|
519
|
+
|
|
520
|
+
NONE =
|
|
521
|
+
T.let(
|
|
522
|
+
:none,
|
|
523
|
+
ContextDev::WebScrapeParams::ImageParams::Dedupe::TaggedSymbol
|
|
524
|
+
)
|
|
525
|
+
VISUAL =
|
|
526
|
+
T.let(
|
|
527
|
+
:visual,
|
|
528
|
+
ContextDev::WebScrapeParams::ImageParams::Dedupe::TaggedSymbol
|
|
529
|
+
)
|
|
530
|
+
|
|
531
|
+
sig do
|
|
532
|
+
override.returns(
|
|
533
|
+
T::Array[
|
|
534
|
+
ContextDev::WebScrapeParams::ImageParams::Dedupe::TaggedSymbol
|
|
535
|
+
]
|
|
536
|
+
)
|
|
537
|
+
end
|
|
538
|
+
def self.values
|
|
539
|
+
end
|
|
540
|
+
end
|
|
541
|
+
|
|
542
|
+
module Enrich
|
|
543
|
+
extend ContextDev::Internal::Type::Enum
|
|
544
|
+
|
|
545
|
+
TaggedSymbol =
|
|
546
|
+
T.type_alias do
|
|
547
|
+
T.all(Symbol, ContextDev::WebScrapeParams::ImageParams::Enrich)
|
|
548
|
+
end
|
|
549
|
+
OrSymbol = T.type_alias { T.any(Symbol, String) }
|
|
550
|
+
|
|
551
|
+
DIMENSIONS =
|
|
552
|
+
T.let(
|
|
553
|
+
:dimensions,
|
|
554
|
+
ContextDev::WebScrapeParams::ImageParams::Enrich::TaggedSymbol
|
|
555
|
+
)
|
|
556
|
+
CLASSIFICATION =
|
|
557
|
+
T.let(
|
|
558
|
+
:classification,
|
|
559
|
+
ContextDev::WebScrapeParams::ImageParams::Enrich::TaggedSymbol
|
|
560
|
+
)
|
|
561
|
+
FILE =
|
|
562
|
+
T.let(
|
|
563
|
+
:file,
|
|
564
|
+
ContextDev::WebScrapeParams::ImageParams::Enrich::TaggedSymbol
|
|
565
|
+
)
|
|
566
|
+
|
|
567
|
+
sig do
|
|
568
|
+
override.returns(
|
|
569
|
+
T::Array[
|
|
570
|
+
ContextDev::WebScrapeParams::ImageParams::Enrich::TaggedSymbol
|
|
571
|
+
]
|
|
572
|
+
)
|
|
573
|
+
end
|
|
574
|
+
def self.values
|
|
575
|
+
end
|
|
576
|
+
end
|
|
577
|
+
end
|
|
578
|
+
|
|
579
|
+
class JsonParams < ContextDev::Internal::Type::BaseModel
|
|
580
|
+
OrHash =
|
|
581
|
+
T.type_alias do
|
|
582
|
+
T.any(
|
|
583
|
+
ContextDev::WebScrapeParams::JsonParams,
|
|
584
|
+
ContextDev::Internal::AnyHash
|
|
585
|
+
)
|
|
586
|
+
end
|
|
587
|
+
|
|
588
|
+
# JSON Schema for the returned object. Must describe a top-level object; at most
|
|
589
|
+
# 50 KB serialized. Optional fields the page does not state are omitted, or null
|
|
590
|
+
# when their type allows null, while required non-nullable fields always receive a
|
|
591
|
+
# best-effort value, so prefer nullable or optional fields for data a page may
|
|
592
|
+
# omit. Zod users can pass the output of z.toJSONSchema().
|
|
593
|
+
sig { returns(T::Hash[Symbol, T.anything]) }
|
|
594
|
+
attr_accessor :schema
|
|
595
|
+
|
|
596
|
+
# Optional guidance on which facts to prioritize or how to interpret schema
|
|
597
|
+
# fields.
|
|
598
|
+
sig { returns(T.nilable(String)) }
|
|
599
|
+
attr_reader :instructions
|
|
600
|
+
|
|
601
|
+
sig { params(instructions: String).void }
|
|
602
|
+
attr_writer :instructions
|
|
603
|
+
|
|
604
|
+
# Required when formats.json is true.
|
|
605
|
+
sig do
|
|
606
|
+
params(
|
|
607
|
+
schema: T::Hash[Symbol, T.anything],
|
|
608
|
+
instructions: String
|
|
609
|
+
).returns(T.attached_class)
|
|
610
|
+
end
|
|
611
|
+
def self.new(
|
|
612
|
+
# JSON Schema for the returned object. Must describe a top-level object; at most
|
|
613
|
+
# 50 KB serialized. Optional fields the page does not state are omitted, or null
|
|
614
|
+
# when their type allows null, while required non-nullable fields always receive a
|
|
615
|
+
# best-effort value, so prefer nullable or optional fields for data a page may
|
|
616
|
+
# omit. Zod users can pass the output of z.toJSONSchema().
|
|
617
|
+
schema:,
|
|
618
|
+
# Optional guidance on which facts to prioritize or how to interpret schema
|
|
619
|
+
# fields.
|
|
620
|
+
instructions: nil
|
|
621
|
+
)
|
|
622
|
+
end
|
|
623
|
+
|
|
624
|
+
sig do
|
|
625
|
+
override.returns(
|
|
626
|
+
{ schema: T::Hash[Symbol, T.anything], instructions: String }
|
|
627
|
+
)
|
|
628
|
+
end
|
|
629
|
+
def to_hash
|
|
630
|
+
end
|
|
631
|
+
end
|
|
632
|
+
|
|
633
|
+
class MarkdownParams < ContextDev::Internal::Type::BaseModel
|
|
634
|
+
OrHash =
|
|
635
|
+
T.type_alias do
|
|
636
|
+
T.any(
|
|
637
|
+
ContextDev::WebScrapeParams::MarkdownParams,
|
|
638
|
+
ContextDev::Internal::AnyHash
|
|
639
|
+
)
|
|
640
|
+
end
|
|
641
|
+
|
|
642
|
+
sig { returns(T.nilable(T::Boolean)) }
|
|
643
|
+
attr_reader :include_images
|
|
644
|
+
|
|
645
|
+
sig { params(include_images: T::Boolean).void }
|
|
646
|
+
attr_writer :include_images
|
|
647
|
+
|
|
648
|
+
sig { returns(T.nilable(T::Boolean)) }
|
|
649
|
+
attr_reader :include_links
|
|
650
|
+
|
|
651
|
+
sig { params(include_links: T::Boolean).void }
|
|
652
|
+
attr_writer :include_links
|
|
653
|
+
|
|
654
|
+
# Base64 images use placeholders by default. Requires includeImages: true.
|
|
655
|
+
sig do
|
|
656
|
+
returns(
|
|
657
|
+
T.nilable(
|
|
658
|
+
ContextDev::WebScrapeParams::MarkdownParams::InlineImages::OrSymbol
|
|
659
|
+
)
|
|
660
|
+
)
|
|
661
|
+
end
|
|
662
|
+
attr_reader :inline_images
|
|
663
|
+
|
|
664
|
+
sig do
|
|
665
|
+
params(
|
|
666
|
+
inline_images:
|
|
667
|
+
ContextDev::WebScrapeParams::MarkdownParams::InlineImages::OrSymbol
|
|
668
|
+
).void
|
|
669
|
+
end
|
|
670
|
+
attr_writer :inline_images
|
|
671
|
+
|
|
672
|
+
# Markdown options. Requires formats.markdown: true.
|
|
673
|
+
sig do
|
|
674
|
+
params(
|
|
675
|
+
include_images: T::Boolean,
|
|
676
|
+
include_links: T::Boolean,
|
|
677
|
+
inline_images:
|
|
678
|
+
ContextDev::WebScrapeParams::MarkdownParams::InlineImages::OrSymbol
|
|
679
|
+
).returns(T.attached_class)
|
|
680
|
+
end
|
|
681
|
+
def self.new(
|
|
682
|
+
include_images: nil,
|
|
683
|
+
include_links: nil,
|
|
684
|
+
# Base64 images use placeholders by default. Requires includeImages: true.
|
|
685
|
+
inline_images: nil
|
|
686
|
+
)
|
|
687
|
+
end
|
|
688
|
+
|
|
689
|
+
sig do
|
|
690
|
+
override.returns(
|
|
691
|
+
{
|
|
692
|
+
include_images: T::Boolean,
|
|
693
|
+
include_links: T::Boolean,
|
|
694
|
+
inline_images:
|
|
695
|
+
ContextDev::WebScrapeParams::MarkdownParams::InlineImages::OrSymbol
|
|
696
|
+
}
|
|
697
|
+
)
|
|
698
|
+
end
|
|
699
|
+
def to_hash
|
|
700
|
+
end
|
|
701
|
+
|
|
702
|
+
# Base64 images use placeholders by default. Requires includeImages: true.
|
|
703
|
+
module InlineImages
|
|
704
|
+
extend ContextDev::Internal::Type::Enum
|
|
705
|
+
|
|
706
|
+
TaggedSymbol =
|
|
707
|
+
T.type_alias do
|
|
708
|
+
T.all(
|
|
709
|
+
Symbol,
|
|
710
|
+
ContextDev::WebScrapeParams::MarkdownParams::InlineImages
|
|
711
|
+
)
|
|
712
|
+
end
|
|
713
|
+
OrSymbol = T.type_alias { T.any(Symbol, String) }
|
|
714
|
+
|
|
715
|
+
PLACEHOLDER =
|
|
716
|
+
T.let(
|
|
717
|
+
:placeholder,
|
|
718
|
+
ContextDev::WebScrapeParams::MarkdownParams::InlineImages::TaggedSymbol
|
|
719
|
+
)
|
|
720
|
+
PRESERVE =
|
|
721
|
+
T.let(
|
|
722
|
+
:preserve,
|
|
723
|
+
ContextDev::WebScrapeParams::MarkdownParams::InlineImages::TaggedSymbol
|
|
724
|
+
)
|
|
725
|
+
|
|
726
|
+
sig do
|
|
727
|
+
override.returns(
|
|
728
|
+
T::Array[
|
|
729
|
+
ContextDev::WebScrapeParams::MarkdownParams::InlineImages::TaggedSymbol
|
|
730
|
+
]
|
|
731
|
+
)
|
|
732
|
+
end
|
|
733
|
+
def self.values
|
|
734
|
+
end
|
|
735
|
+
end
|
|
736
|
+
end
|
|
737
|
+
|
|
738
|
+
class ParseParams < ContextDev::Internal::Type::BaseModel
|
|
739
|
+
OrHash =
|
|
740
|
+
T.type_alias do
|
|
741
|
+
T.any(
|
|
742
|
+
ContextDev::WebScrapeParams::ParseParams,
|
|
743
|
+
ContextDev::Internal::AnyHash
|
|
744
|
+
)
|
|
745
|
+
end
|
|
746
|
+
|
|
747
|
+
# Map field names to CSS selectors or rules. Missing items return null; missing
|
|
748
|
+
# lists return [].
|
|
749
|
+
sig do
|
|
750
|
+
returns(
|
|
751
|
+
T::Hash[
|
|
752
|
+
Symbol,
|
|
753
|
+
T.any(
|
|
754
|
+
String,
|
|
755
|
+
ContextDev::WebScrapeParams::ParseParams::Rule::UnionMember1
|
|
756
|
+
)
|
|
757
|
+
]
|
|
758
|
+
)
|
|
759
|
+
end
|
|
760
|
+
attr_accessor :rules
|
|
761
|
+
|
|
762
|
+
# Required when formats.parse is true.
|
|
763
|
+
sig do
|
|
764
|
+
params(
|
|
765
|
+
rules:
|
|
766
|
+
T::Hash[
|
|
767
|
+
Symbol,
|
|
768
|
+
T.any(
|
|
769
|
+
String,
|
|
770
|
+
ContextDev::WebScrapeParams::ParseParams::Rule::UnionMember1::OrHash
|
|
771
|
+
)
|
|
772
|
+
]
|
|
773
|
+
).returns(T.attached_class)
|
|
774
|
+
end
|
|
775
|
+
def self.new(
|
|
776
|
+
# Map field names to CSS selectors or rules. Missing items return null; missing
|
|
777
|
+
# lists return [].
|
|
778
|
+
rules:
|
|
779
|
+
)
|
|
780
|
+
end
|
|
781
|
+
|
|
782
|
+
sig do
|
|
783
|
+
override.returns(
|
|
784
|
+
{
|
|
785
|
+
rules:
|
|
786
|
+
T::Hash[
|
|
787
|
+
Symbol,
|
|
788
|
+
T.any(
|
|
789
|
+
String,
|
|
790
|
+
ContextDev::WebScrapeParams::ParseParams::Rule::UnionMember1
|
|
791
|
+
)
|
|
792
|
+
]
|
|
793
|
+
}
|
|
794
|
+
)
|
|
795
|
+
end
|
|
796
|
+
def to_hash
|
|
797
|
+
end
|
|
798
|
+
|
|
799
|
+
module Rule
|
|
800
|
+
extend ContextDev::Internal::Type::Union
|
|
801
|
+
|
|
802
|
+
Variants =
|
|
803
|
+
T.type_alias do
|
|
804
|
+
T.any(
|
|
805
|
+
String,
|
|
806
|
+
ContextDev::WebScrapeParams::ParseParams::Rule::UnionMember1
|
|
807
|
+
)
|
|
808
|
+
end
|
|
809
|
+
|
|
810
|
+
class UnionMember1 < ContextDev::Internal::Type::BaseModel
|
|
811
|
+
OrHash =
|
|
812
|
+
T.type_alias do
|
|
813
|
+
T.any(
|
|
814
|
+
ContextDev::WebScrapeParams::ParseParams::Rule::UnionMember1,
|
|
815
|
+
ContextDev::Internal::AnyHash
|
|
816
|
+
)
|
|
817
|
+
end
|
|
818
|
+
|
|
819
|
+
sig { returns(String) }
|
|
820
|
+
attr_accessor :selector
|
|
821
|
+
|
|
822
|
+
sig do
|
|
823
|
+
returns(
|
|
824
|
+
T.nilable(
|
|
825
|
+
T.any(
|
|
826
|
+
ContextDev::WebScrapeParams::ParseParams::Rule::UnionMember1::Output::OrSymbol,
|
|
827
|
+
String,
|
|
828
|
+
T.anything
|
|
829
|
+
)
|
|
830
|
+
)
|
|
831
|
+
)
|
|
832
|
+
end
|
|
833
|
+
attr_reader :output
|
|
834
|
+
|
|
835
|
+
sig do
|
|
836
|
+
params(
|
|
837
|
+
output:
|
|
838
|
+
T.any(
|
|
839
|
+
ContextDev::WebScrapeParams::ParseParams::Rule::UnionMember1::Output::OrSymbol,
|
|
840
|
+
String,
|
|
841
|
+
T.anything
|
|
842
|
+
)
|
|
843
|
+
).void
|
|
844
|
+
end
|
|
845
|
+
attr_writer :output
|
|
846
|
+
|
|
847
|
+
sig do
|
|
848
|
+
returns(
|
|
849
|
+
T.nilable(
|
|
850
|
+
ContextDev::WebScrapeParams::ParseParams::Rule::UnionMember1::Type::OrSymbol
|
|
851
|
+
)
|
|
852
|
+
)
|
|
853
|
+
end
|
|
854
|
+
attr_reader :type
|
|
855
|
+
|
|
856
|
+
sig do
|
|
857
|
+
params(
|
|
858
|
+
type:
|
|
859
|
+
ContextDev::WebScrapeParams::ParseParams::Rule::UnionMember1::Type::OrSymbol
|
|
860
|
+
).void
|
|
861
|
+
end
|
|
862
|
+
attr_writer :type
|
|
863
|
+
|
|
864
|
+
sig do
|
|
865
|
+
params(
|
|
866
|
+
selector: String,
|
|
867
|
+
output:
|
|
868
|
+
T.any(
|
|
869
|
+
ContextDev::WebScrapeParams::ParseParams::Rule::UnionMember1::Output::OrSymbol,
|
|
870
|
+
String,
|
|
871
|
+
T.anything
|
|
872
|
+
),
|
|
873
|
+
type:
|
|
874
|
+
ContextDev::WebScrapeParams::ParseParams::Rule::UnionMember1::Type::OrSymbol
|
|
875
|
+
).returns(T.attached_class)
|
|
876
|
+
end
|
|
877
|
+
def self.new(selector:, output: nil, type: nil)
|
|
878
|
+
end
|
|
879
|
+
|
|
880
|
+
sig do
|
|
881
|
+
override.returns(
|
|
882
|
+
{
|
|
883
|
+
selector: String,
|
|
884
|
+
output:
|
|
885
|
+
T.any(
|
|
886
|
+
ContextDev::WebScrapeParams::ParseParams::Rule::UnionMember1::Output::OrSymbol,
|
|
887
|
+
String,
|
|
888
|
+
T.anything
|
|
889
|
+
),
|
|
890
|
+
type:
|
|
891
|
+
ContextDev::WebScrapeParams::ParseParams::Rule::UnionMember1::Type::OrSymbol
|
|
892
|
+
}
|
|
893
|
+
)
|
|
894
|
+
end
|
|
895
|
+
def to_hash
|
|
896
|
+
end
|
|
897
|
+
|
|
898
|
+
module Output
|
|
899
|
+
extend ContextDev::Internal::Type::Union
|
|
900
|
+
|
|
901
|
+
Variants =
|
|
902
|
+
T.type_alias do
|
|
903
|
+
T.any(
|
|
904
|
+
ContextDev::WebScrapeParams::ParseParams::Rule::UnionMember1::Output::TaggedSymbol,
|
|
905
|
+
String,
|
|
906
|
+
T.anything
|
|
907
|
+
)
|
|
908
|
+
end
|
|
909
|
+
|
|
910
|
+
sig do
|
|
911
|
+
override.returns(
|
|
912
|
+
T::Array[
|
|
913
|
+
ContextDev::WebScrapeParams::ParseParams::Rule::UnionMember1::Output::Variants
|
|
914
|
+
]
|
|
915
|
+
)
|
|
916
|
+
end
|
|
917
|
+
def self.variants
|
|
918
|
+
end
|
|
919
|
+
|
|
920
|
+
TaggedSymbol =
|
|
921
|
+
T.type_alias do
|
|
922
|
+
T.all(
|
|
923
|
+
Symbol,
|
|
924
|
+
ContextDev::WebScrapeParams::ParseParams::Rule::UnionMember1::Output
|
|
925
|
+
)
|
|
926
|
+
end
|
|
927
|
+
OrSymbol = T.type_alias { T.any(Symbol, String) }
|
|
928
|
+
|
|
929
|
+
TEXT =
|
|
930
|
+
T.let(
|
|
931
|
+
:text,
|
|
932
|
+
ContextDev::WebScrapeParams::ParseParams::Rule::UnionMember1::Output::TaggedSymbol
|
|
933
|
+
)
|
|
934
|
+
HTML =
|
|
935
|
+
T.let(
|
|
936
|
+
:html,
|
|
937
|
+
ContextDev::WebScrapeParams::ParseParams::Rule::UnionMember1::Output::TaggedSymbol
|
|
938
|
+
)
|
|
939
|
+
end
|
|
940
|
+
|
|
941
|
+
module Type
|
|
942
|
+
extend ContextDev::Internal::Type::Enum
|
|
943
|
+
|
|
944
|
+
TaggedSymbol =
|
|
945
|
+
T.type_alias do
|
|
946
|
+
T.all(
|
|
947
|
+
Symbol,
|
|
948
|
+
ContextDev::WebScrapeParams::ParseParams::Rule::UnionMember1::Type
|
|
949
|
+
)
|
|
950
|
+
end
|
|
951
|
+
OrSymbol = T.type_alias { T.any(Symbol, String) }
|
|
952
|
+
|
|
953
|
+
ITEM =
|
|
954
|
+
T.let(
|
|
955
|
+
:item,
|
|
956
|
+
ContextDev::WebScrapeParams::ParseParams::Rule::UnionMember1::Type::TaggedSymbol
|
|
957
|
+
)
|
|
958
|
+
LIST =
|
|
959
|
+
T.let(
|
|
960
|
+
:list,
|
|
961
|
+
ContextDev::WebScrapeParams::ParseParams::Rule::UnionMember1::Type::TaggedSymbol
|
|
962
|
+
)
|
|
963
|
+
|
|
964
|
+
sig do
|
|
965
|
+
override.returns(
|
|
966
|
+
T::Array[
|
|
967
|
+
ContextDev::WebScrapeParams::ParseParams::Rule::UnionMember1::Type::TaggedSymbol
|
|
968
|
+
]
|
|
969
|
+
)
|
|
970
|
+
end
|
|
971
|
+
def self.values
|
|
972
|
+
end
|
|
973
|
+
end
|
|
974
|
+
end
|
|
975
|
+
|
|
976
|
+
sig do
|
|
977
|
+
override.returns(
|
|
978
|
+
T::Array[ContextDev::WebScrapeParams::ParseParams::Rule::Variants]
|
|
979
|
+
)
|
|
980
|
+
end
|
|
981
|
+
def self.variants
|
|
982
|
+
end
|
|
983
|
+
end
|
|
984
|
+
end
|
|
985
|
+
|
|
986
|
+
class ProductParams < ContextDev::Internal::Type::BaseModel
|
|
987
|
+
OrHash =
|
|
988
|
+
T.type_alias do
|
|
989
|
+
T.any(
|
|
990
|
+
ContextDev::WebScrapeParams::ProductParams,
|
|
991
|
+
ContextDev::Internal::AnyHash
|
|
992
|
+
)
|
|
993
|
+
end
|
|
994
|
+
|
|
995
|
+
# Extract the product with a specialized model when the page has no structured
|
|
996
|
+
# product data. Adds six credits when the model returns a verdict. If the fallback
|
|
997
|
+
# fails, returns a partial response with the deterministic result and no fallback
|
|
998
|
+
# charge. Request deadlines and client disconnects still apply.
|
|
999
|
+
sig { returns(T.nilable(T::Boolean)) }
|
|
1000
|
+
attr_reader :use_ai_fallback
|
|
1001
|
+
|
|
1002
|
+
sig { params(use_ai_fallback: T::Boolean).void }
|
|
1003
|
+
attr_writer :use_ai_fallback
|
|
1004
|
+
|
|
1005
|
+
# Product options. Requires formats.product: true.
|
|
1006
|
+
sig { params(use_ai_fallback: T::Boolean).returns(T.attached_class) }
|
|
1007
|
+
def self.new(
|
|
1008
|
+
# Extract the product with a specialized model when the page has no structured
|
|
1009
|
+
# product data. Adds six credits when the model returns a verdict. If the fallback
|
|
1010
|
+
# fails, returns a partial response with the deterministic result and no fallback
|
|
1011
|
+
# charge. Request deadlines and client disconnects still apply.
|
|
1012
|
+
use_ai_fallback: nil
|
|
1013
|
+
)
|
|
1014
|
+
end
|
|
1015
|
+
|
|
1016
|
+
sig { override.returns({ use_ai_fallback: T::Boolean }) }
|
|
1017
|
+
def to_hash
|
|
1018
|
+
end
|
|
1019
|
+
end
|
|
1020
|
+
|
|
1021
|
+
class ScreenshotParams < ContextDev::Internal::Type::BaseModel
|
|
1022
|
+
OrHash =
|
|
1023
|
+
T.type_alias do
|
|
1024
|
+
T.any(
|
|
1025
|
+
ContextDev::WebScrapeParams::ScreenshotParams,
|
|
1026
|
+
ContextDev::Internal::AnyHash
|
|
1027
|
+
)
|
|
1028
|
+
end
|
|
1029
|
+
|
|
1030
|
+
# Viewport, full page, one visible element, or a rectangle. Maximum 40 megapixels.
|
|
1031
|
+
sig do
|
|
1032
|
+
returns(
|
|
1033
|
+
T.nilable(
|
|
1034
|
+
T.any(
|
|
1035
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Area::Page::OrSymbol,
|
|
1036
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Area::Element,
|
|
1037
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Area::Rectangle
|
|
1038
|
+
)
|
|
1039
|
+
)
|
|
1040
|
+
)
|
|
1041
|
+
end
|
|
1042
|
+
attr_reader :area
|
|
1043
|
+
|
|
1044
|
+
sig do
|
|
1045
|
+
params(
|
|
1046
|
+
area:
|
|
1047
|
+
T.any(
|
|
1048
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Area::Page::OrSymbol,
|
|
1049
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Area::Element::OrHash,
|
|
1050
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Area::Rectangle::OrHash
|
|
1051
|
+
)
|
|
1052
|
+
).void
|
|
1053
|
+
end
|
|
1054
|
+
attr_writer :area
|
|
1055
|
+
|
|
1056
|
+
sig do
|
|
1057
|
+
returns(
|
|
1058
|
+
T.nilable(
|
|
1059
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Format::OrSymbol
|
|
1060
|
+
)
|
|
1061
|
+
)
|
|
1062
|
+
end
|
|
1063
|
+
attr_reader :format_
|
|
1064
|
+
|
|
1065
|
+
sig do
|
|
1066
|
+
params(
|
|
1067
|
+
format_:
|
|
1068
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Format::OrSymbol
|
|
1069
|
+
).void
|
|
1070
|
+
end
|
|
1071
|
+
attr_writer :format_
|
|
1072
|
+
|
|
1073
|
+
# Screenshot options. Requires formats.screenshot: true.
|
|
1074
|
+
sig do
|
|
1075
|
+
params(
|
|
1076
|
+
area:
|
|
1077
|
+
T.any(
|
|
1078
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Area::Page::OrSymbol,
|
|
1079
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Area::Element::OrHash,
|
|
1080
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Area::Rectangle::OrHash
|
|
1081
|
+
),
|
|
1082
|
+
format_:
|
|
1083
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Format::OrSymbol
|
|
1084
|
+
).returns(T.attached_class)
|
|
1085
|
+
end
|
|
1086
|
+
def self.new(
|
|
1087
|
+
# Viewport, full page, one visible element, or a rectangle. Maximum 40 megapixels.
|
|
1088
|
+
area: nil,
|
|
1089
|
+
format_: nil
|
|
1090
|
+
)
|
|
1091
|
+
end
|
|
1092
|
+
|
|
1093
|
+
sig do
|
|
1094
|
+
override.returns(
|
|
1095
|
+
{
|
|
1096
|
+
area:
|
|
1097
|
+
T.any(
|
|
1098
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Area::Page::OrSymbol,
|
|
1099
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Area::Element,
|
|
1100
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Area::Rectangle
|
|
1101
|
+
),
|
|
1102
|
+
format_:
|
|
1103
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Format::OrSymbol
|
|
1104
|
+
}
|
|
1105
|
+
)
|
|
1106
|
+
end
|
|
1107
|
+
def to_hash
|
|
1108
|
+
end
|
|
1109
|
+
|
|
1110
|
+
# Viewport, full page, one visible element, or a rectangle. Maximum 40 megapixels.
|
|
1111
|
+
module Area
|
|
1112
|
+
extend ContextDev::Internal::Type::Union
|
|
1113
|
+
|
|
1114
|
+
Variants =
|
|
1115
|
+
T.type_alias do
|
|
1116
|
+
T.any(
|
|
1117
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Area::Page::TaggedSymbol,
|
|
1118
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Area::Element,
|
|
1119
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Area::Rectangle
|
|
1120
|
+
)
|
|
1121
|
+
end
|
|
1122
|
+
|
|
1123
|
+
module Page
|
|
1124
|
+
extend ContextDev::Internal::Type::Enum
|
|
1125
|
+
|
|
1126
|
+
TaggedSymbol =
|
|
1127
|
+
T.type_alias do
|
|
1128
|
+
T.all(
|
|
1129
|
+
Symbol,
|
|
1130
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Area::Page
|
|
1131
|
+
)
|
|
1132
|
+
end
|
|
1133
|
+
OrSymbol = T.type_alias { T.any(Symbol, String) }
|
|
1134
|
+
|
|
1135
|
+
VIEWPORT =
|
|
1136
|
+
T.let(
|
|
1137
|
+
:viewport,
|
|
1138
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Area::Page::TaggedSymbol
|
|
1139
|
+
)
|
|
1140
|
+
FULL_PAGE =
|
|
1141
|
+
T.let(
|
|
1142
|
+
:fullPage,
|
|
1143
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Area::Page::TaggedSymbol
|
|
1144
|
+
)
|
|
1145
|
+
|
|
1146
|
+
sig do
|
|
1147
|
+
override.returns(
|
|
1148
|
+
T::Array[
|
|
1149
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Area::Page::TaggedSymbol
|
|
1150
|
+
]
|
|
1151
|
+
)
|
|
1152
|
+
end
|
|
1153
|
+
def self.values
|
|
1154
|
+
end
|
|
1155
|
+
end
|
|
1156
|
+
|
|
1157
|
+
class Element < ContextDev::Internal::Type::BaseModel
|
|
1158
|
+
OrHash =
|
|
1159
|
+
T.type_alias do
|
|
1160
|
+
T.any(
|
|
1161
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Area::Element,
|
|
1162
|
+
ContextDev::Internal::AnyHash
|
|
1163
|
+
)
|
|
1164
|
+
end
|
|
1165
|
+
|
|
1166
|
+
# Must match one visible element.
|
|
1167
|
+
sig { returns(String) }
|
|
1168
|
+
attr_accessor :selector
|
|
1169
|
+
|
|
1170
|
+
sig { params(selector: String).returns(T.attached_class) }
|
|
1171
|
+
def self.new(
|
|
1172
|
+
# Must match one visible element.
|
|
1173
|
+
selector:
|
|
1174
|
+
)
|
|
1175
|
+
end
|
|
1176
|
+
|
|
1177
|
+
sig { override.returns({ selector: String }) }
|
|
1178
|
+
def to_hash
|
|
1179
|
+
end
|
|
1180
|
+
end
|
|
1181
|
+
|
|
1182
|
+
class Rectangle < ContextDev::Internal::Type::BaseModel
|
|
1183
|
+
OrHash =
|
|
1184
|
+
T.type_alias do
|
|
1185
|
+
T.any(
|
|
1186
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Area::Rectangle,
|
|
1187
|
+
ContextDev::Internal::AnyHash
|
|
1188
|
+
)
|
|
1189
|
+
end
|
|
1190
|
+
|
|
1191
|
+
sig { returns(Integer) }
|
|
1192
|
+
attr_accessor :height
|
|
1193
|
+
|
|
1194
|
+
sig { returns(Integer) }
|
|
1195
|
+
attr_accessor :width
|
|
1196
|
+
|
|
1197
|
+
sig { returns(Integer) }
|
|
1198
|
+
attr_accessor :x
|
|
1199
|
+
|
|
1200
|
+
sig { returns(Integer) }
|
|
1201
|
+
attr_accessor :y_
|
|
1202
|
+
|
|
1203
|
+
# Pixels from the document origin.
|
|
1204
|
+
sig do
|
|
1205
|
+
params(
|
|
1206
|
+
height: Integer,
|
|
1207
|
+
width: Integer,
|
|
1208
|
+
x: Integer,
|
|
1209
|
+
y_: Integer
|
|
1210
|
+
).returns(T.attached_class)
|
|
1211
|
+
end
|
|
1212
|
+
def self.new(height:, width:, x:, y_:)
|
|
1213
|
+
end
|
|
1214
|
+
|
|
1215
|
+
sig do
|
|
1216
|
+
override.returns(
|
|
1217
|
+
{ height: Integer, width: Integer, x: Integer, y_: Integer }
|
|
1218
|
+
)
|
|
1219
|
+
end
|
|
1220
|
+
def to_hash
|
|
1221
|
+
end
|
|
1222
|
+
end
|
|
1223
|
+
|
|
1224
|
+
sig do
|
|
1225
|
+
override.returns(
|
|
1226
|
+
T::Array[
|
|
1227
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Area::Variants
|
|
1228
|
+
]
|
|
1229
|
+
)
|
|
1230
|
+
end
|
|
1231
|
+
def self.variants
|
|
1232
|
+
end
|
|
1233
|
+
end
|
|
1234
|
+
|
|
1235
|
+
module Format
|
|
1236
|
+
extend ContextDev::Internal::Type::Enum
|
|
1237
|
+
|
|
1238
|
+
TaggedSymbol =
|
|
1239
|
+
T.type_alias do
|
|
1240
|
+
T.all(
|
|
1241
|
+
Symbol,
|
|
1242
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Format
|
|
1243
|
+
)
|
|
1244
|
+
end
|
|
1245
|
+
OrSymbol = T.type_alias { T.any(Symbol, String) }
|
|
1246
|
+
|
|
1247
|
+
PNG =
|
|
1248
|
+
T.let(
|
|
1249
|
+
:png,
|
|
1250
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Format::TaggedSymbol
|
|
1251
|
+
)
|
|
1252
|
+
JPEG =
|
|
1253
|
+
T.let(
|
|
1254
|
+
:jpeg,
|
|
1255
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Format::TaggedSymbol
|
|
1256
|
+
)
|
|
1257
|
+
WEBP =
|
|
1258
|
+
T.let(
|
|
1259
|
+
:webp,
|
|
1260
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Format::TaggedSymbol
|
|
1261
|
+
)
|
|
1262
|
+
|
|
1263
|
+
sig do
|
|
1264
|
+
override.returns(
|
|
1265
|
+
T::Array[
|
|
1266
|
+
ContextDev::WebScrapeParams::ScreenshotParams::Format::TaggedSymbol
|
|
1267
|
+
]
|
|
1268
|
+
)
|
|
1269
|
+
end
|
|
1270
|
+
def self.values
|
|
1271
|
+
end
|
|
1272
|
+
end
|
|
1273
|
+
end
|
|
1274
|
+
|
|
1275
|
+
class SharedParams < ContextDev::Internal::Type::BaseModel
|
|
1276
|
+
OrHash =
|
|
1277
|
+
T.type_alias do
|
|
1278
|
+
T.any(
|
|
1279
|
+
ContextDev::WebScrapeParams::SharedParams,
|
|
1280
|
+
ContextDev::Internal::AnyHash
|
|
1281
|
+
)
|
|
1282
|
+
end
|
|
1283
|
+
|
|
1284
|
+
# Run in order before capture. A failed action fails the request. Bypasses
|
|
1285
|
+
# caching.
|
|
1286
|
+
sig do
|
|
1287
|
+
returns(
|
|
1288
|
+
T.nilable(
|
|
1289
|
+
T::Array[
|
|
1290
|
+
T.any(
|
|
1291
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Perform,
|
|
1292
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Scroll,
|
|
1293
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Wait,
|
|
1294
|
+
ContextDev::WebScrapeParams::SharedParams::Action::WaitFor
|
|
1295
|
+
)
|
|
1296
|
+
]
|
|
1297
|
+
)
|
|
1298
|
+
)
|
|
1299
|
+
end
|
|
1300
|
+
attr_reader :actions
|
|
1301
|
+
|
|
1302
|
+
sig do
|
|
1303
|
+
params(
|
|
1304
|
+
actions:
|
|
1305
|
+
T::Array[
|
|
1306
|
+
T.any(
|
|
1307
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Perform::OrHash,
|
|
1308
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Scroll::OrHash,
|
|
1309
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Wait::OrHash,
|
|
1310
|
+
ContextDev::WebScrapeParams::SharedParams::Action::WaitFor::OrHash
|
|
1311
|
+
)
|
|
1312
|
+
]
|
|
1313
|
+
).void
|
|
1314
|
+
end
|
|
1315
|
+
attr_writer :actions
|
|
1316
|
+
|
|
1317
|
+
# Supported two-letter country code, case-insensitive. Applies to every output,
|
|
1318
|
+
# including image downloads.
|
|
1319
|
+
sig { returns(T.nilable(String)) }
|
|
1320
|
+
attr_reader :country
|
|
1321
|
+
|
|
1322
|
+
sig { params(country: String).void }
|
|
1323
|
+
attr_writer :country
|
|
1324
|
+
|
|
1325
|
+
# Dismiss cookie banners by accepting cookies before actions.
|
|
1326
|
+
sig { returns(T.nilable(T::Boolean)) }
|
|
1327
|
+
attr_reader :dismiss_cookies
|
|
1328
|
+
|
|
1329
|
+
sig { params(dismiss_cookies: T::Boolean).void }
|
|
1330
|
+
attr_writer :dismiss_cookies
|
|
1331
|
+
|
|
1332
|
+
# Dismiss other popups before actions.
|
|
1333
|
+
sig { returns(T.nilable(T::Boolean)) }
|
|
1334
|
+
attr_reader :dismiss_popups
|
|
1335
|
+
|
|
1336
|
+
sig { params(dismiss_popups: T::Boolean).void }
|
|
1337
|
+
attr_writer :dismiss_popups
|
|
1338
|
+
|
|
1339
|
+
# Remove matching content. Exclusions win.
|
|
1340
|
+
sig { returns(T.nilable(T::Array[String])) }
|
|
1341
|
+
attr_reader :exclude_selectors
|
|
1342
|
+
|
|
1343
|
+
sig { params(exclude_selectors: T::Array[String]).void }
|
|
1344
|
+
attr_writer :exclude_selectors
|
|
1345
|
+
|
|
1346
|
+
# Headers for the target origin. Requests with custom headers bypass caching.
|
|
1347
|
+
sig { returns(T.nilable(T::Hash[Symbol, String])) }
|
|
1348
|
+
attr_reader :headers
|
|
1349
|
+
|
|
1350
|
+
sig { params(headers: T::Hash[Symbol, String]).void }
|
|
1351
|
+
attr_writer :headers
|
|
1352
|
+
|
|
1353
|
+
# Include iframe content in extraction. Screenshots show visible frames
|
|
1354
|
+
# regardless.
|
|
1355
|
+
sig { returns(T.nilable(T::Boolean)) }
|
|
1356
|
+
attr_reader :include_frames
|
|
1357
|
+
|
|
1358
|
+
sig { params(include_frames: T::Boolean).void }
|
|
1359
|
+
attr_writer :include_frames
|
|
1360
|
+
|
|
1361
|
+
# Keep matching content after mainContentOnly.
|
|
1362
|
+
sig { returns(T.nilable(T::Array[String])) }
|
|
1363
|
+
attr_reader :include_selectors
|
|
1364
|
+
|
|
1365
|
+
sig { params(include_selectors: T::Array[String]).void }
|
|
1366
|
+
attr_writer :include_selectors
|
|
1367
|
+
|
|
1368
|
+
# Keep only main content in HTML, Markdown, images, and parsed fields.
|
|
1369
|
+
sig { returns(T.nilable(T::Boolean)) }
|
|
1370
|
+
attr_reader :main_content_only
|
|
1371
|
+
|
|
1372
|
+
sig { params(main_content_only: T::Boolean).void }
|
|
1373
|
+
attr_writer :main_content_only
|
|
1374
|
+
|
|
1375
|
+
# Document parsing options.
|
|
1376
|
+
sig do
|
|
1377
|
+
returns(T.nilable(ContextDev::WebScrapeParams::SharedParams::Parsers))
|
|
1378
|
+
end
|
|
1379
|
+
attr_reader :parsers
|
|
1380
|
+
|
|
1381
|
+
sig do
|
|
1382
|
+
params(
|
|
1383
|
+
parsers: ContextDev::WebScrapeParams::SharedParams::Parsers::OrHash
|
|
1384
|
+
).void
|
|
1385
|
+
end
|
|
1386
|
+
attr_writer :parsers
|
|
1387
|
+
|
|
1388
|
+
# Settle animations before capture. Defaults to true with screenshots, otherwise
|
|
1389
|
+
# false.
|
|
1390
|
+
sig { returns(T.nilable(T::Boolean)) }
|
|
1391
|
+
attr_reader :settle_animations
|
|
1392
|
+
|
|
1393
|
+
sig { params(settle_animations: T::Boolean).void }
|
|
1394
|
+
attr_writer :settle_animations
|
|
1395
|
+
|
|
1396
|
+
# Override the browser color scheme.
|
|
1397
|
+
sig do
|
|
1398
|
+
returns(
|
|
1399
|
+
T.nilable(
|
|
1400
|
+
ContextDev::WebScrapeParams::SharedParams::Theme::OrSymbol
|
|
1401
|
+
)
|
|
1402
|
+
)
|
|
1403
|
+
end
|
|
1404
|
+
attr_reader :theme
|
|
1405
|
+
|
|
1406
|
+
sig do
|
|
1407
|
+
params(
|
|
1408
|
+
theme: ContextDev::WebScrapeParams::SharedParams::Theme::OrSymbol
|
|
1409
|
+
).void
|
|
1410
|
+
end
|
|
1411
|
+
attr_writer :theme
|
|
1412
|
+
|
|
1413
|
+
# Browser dimensions in pixels.
|
|
1414
|
+
sig do
|
|
1415
|
+
returns(
|
|
1416
|
+
T.nilable(ContextDev::WebScrapeParams::SharedParams::Viewport)
|
|
1417
|
+
)
|
|
1418
|
+
end
|
|
1419
|
+
attr_reader :viewport
|
|
1420
|
+
|
|
1421
|
+
sig do
|
|
1422
|
+
params(
|
|
1423
|
+
viewport:
|
|
1424
|
+
ContextDev::WebScrapeParams::SharedParams::Viewport::OrHash
|
|
1425
|
+
).void
|
|
1426
|
+
end
|
|
1427
|
+
attr_writer :viewport
|
|
1428
|
+
|
|
1429
|
+
# After actions, wait this many milliseconds or until a CSS selector is visible.
|
|
1430
|
+
# Defaults to 500 ms, or 2000 ms with frames or an XML URL. Set 0 to skip.
|
|
1431
|
+
sig do
|
|
1432
|
+
returns(
|
|
1433
|
+
T.nilable(
|
|
1434
|
+
ContextDev::WebScrapeParams::SharedParams::WaitFor::Variants
|
|
1435
|
+
)
|
|
1436
|
+
)
|
|
1437
|
+
end
|
|
1438
|
+
attr_reader :wait_for
|
|
1439
|
+
|
|
1440
|
+
sig do
|
|
1441
|
+
params(
|
|
1442
|
+
wait_for:
|
|
1443
|
+
ContextDev::WebScrapeParams::SharedParams::WaitFor::Variants
|
|
1444
|
+
).void
|
|
1445
|
+
end
|
|
1446
|
+
attr_writer :wait_for
|
|
1447
|
+
|
|
1448
|
+
# Shared browser and content settings. Content filters leave screenshots and
|
|
1449
|
+
# original bytes unchanged.
|
|
1450
|
+
sig do
|
|
1451
|
+
params(
|
|
1452
|
+
actions:
|
|
1453
|
+
T::Array[
|
|
1454
|
+
T.any(
|
|
1455
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Perform::OrHash,
|
|
1456
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Scroll::OrHash,
|
|
1457
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Wait::OrHash,
|
|
1458
|
+
ContextDev::WebScrapeParams::SharedParams::Action::WaitFor::OrHash
|
|
1459
|
+
)
|
|
1460
|
+
],
|
|
1461
|
+
country: String,
|
|
1462
|
+
dismiss_cookies: T::Boolean,
|
|
1463
|
+
dismiss_popups: T::Boolean,
|
|
1464
|
+
exclude_selectors: T::Array[String],
|
|
1465
|
+
headers: T::Hash[Symbol, String],
|
|
1466
|
+
include_frames: T::Boolean,
|
|
1467
|
+
include_selectors: T::Array[String],
|
|
1468
|
+
main_content_only: T::Boolean,
|
|
1469
|
+
parsers: ContextDev::WebScrapeParams::SharedParams::Parsers::OrHash,
|
|
1470
|
+
settle_animations: T::Boolean,
|
|
1471
|
+
theme: ContextDev::WebScrapeParams::SharedParams::Theme::OrSymbol,
|
|
1472
|
+
viewport:
|
|
1473
|
+
ContextDev::WebScrapeParams::SharedParams::Viewport::OrHash,
|
|
1474
|
+
wait_for:
|
|
1475
|
+
ContextDev::WebScrapeParams::SharedParams::WaitFor::Variants
|
|
1476
|
+
).returns(T.attached_class)
|
|
1477
|
+
end
|
|
1478
|
+
def self.new(
|
|
1479
|
+
# Run in order before capture. A failed action fails the request. Bypasses
|
|
1480
|
+
# caching.
|
|
1481
|
+
actions: nil,
|
|
1482
|
+
# Supported two-letter country code, case-insensitive. Applies to every output,
|
|
1483
|
+
# including image downloads.
|
|
1484
|
+
country: nil,
|
|
1485
|
+
# Dismiss cookie banners by accepting cookies before actions.
|
|
1486
|
+
dismiss_cookies: nil,
|
|
1487
|
+
# Dismiss other popups before actions.
|
|
1488
|
+
dismiss_popups: nil,
|
|
1489
|
+
# Remove matching content. Exclusions win.
|
|
1490
|
+
exclude_selectors: nil,
|
|
1491
|
+
# Headers for the target origin. Requests with custom headers bypass caching.
|
|
1492
|
+
headers: nil,
|
|
1493
|
+
# Include iframe content in extraction. Screenshots show visible frames
|
|
1494
|
+
# regardless.
|
|
1495
|
+
include_frames: nil,
|
|
1496
|
+
# Keep matching content after mainContentOnly.
|
|
1497
|
+
include_selectors: nil,
|
|
1498
|
+
# Keep only main content in HTML, Markdown, images, and parsed fields.
|
|
1499
|
+
main_content_only: nil,
|
|
1500
|
+
# Document parsing options.
|
|
1501
|
+
parsers: nil,
|
|
1502
|
+
# Settle animations before capture. Defaults to true with screenshots, otherwise
|
|
1503
|
+
# false.
|
|
1504
|
+
settle_animations: nil,
|
|
1505
|
+
# Override the browser color scheme.
|
|
1506
|
+
theme: nil,
|
|
1507
|
+
# Browser dimensions in pixels.
|
|
1508
|
+
viewport: nil,
|
|
1509
|
+
# After actions, wait this many milliseconds or until a CSS selector is visible.
|
|
1510
|
+
# Defaults to 500 ms, or 2000 ms with frames or an XML URL. Set 0 to skip.
|
|
1511
|
+
wait_for: nil
|
|
1512
|
+
)
|
|
1513
|
+
end
|
|
1514
|
+
|
|
1515
|
+
sig do
|
|
1516
|
+
override.returns(
|
|
1517
|
+
{
|
|
1518
|
+
actions:
|
|
1519
|
+
T::Array[
|
|
1520
|
+
T.any(
|
|
1521
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Perform,
|
|
1522
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Scroll,
|
|
1523
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Wait,
|
|
1524
|
+
ContextDev::WebScrapeParams::SharedParams::Action::WaitFor
|
|
1525
|
+
)
|
|
1526
|
+
],
|
|
1527
|
+
country: String,
|
|
1528
|
+
dismiss_cookies: T::Boolean,
|
|
1529
|
+
dismiss_popups: T::Boolean,
|
|
1530
|
+
exclude_selectors: T::Array[String],
|
|
1531
|
+
headers: T::Hash[Symbol, String],
|
|
1532
|
+
include_frames: T::Boolean,
|
|
1533
|
+
include_selectors: T::Array[String],
|
|
1534
|
+
main_content_only: T::Boolean,
|
|
1535
|
+
parsers: ContextDev::WebScrapeParams::SharedParams::Parsers,
|
|
1536
|
+
settle_animations: T::Boolean,
|
|
1537
|
+
theme: ContextDev::WebScrapeParams::SharedParams::Theme::OrSymbol,
|
|
1538
|
+
viewport: ContextDev::WebScrapeParams::SharedParams::Viewport,
|
|
1539
|
+
wait_for:
|
|
1540
|
+
ContextDev::WebScrapeParams::SharedParams::WaitFor::Variants
|
|
1541
|
+
}
|
|
1542
|
+
)
|
|
1543
|
+
end
|
|
1544
|
+
def to_hash
|
|
1545
|
+
end
|
|
1546
|
+
|
|
1547
|
+
module Action
|
|
1548
|
+
extend ContextDev::Internal::Type::Union
|
|
1549
|
+
|
|
1550
|
+
Variants =
|
|
1551
|
+
T.type_alias do
|
|
1552
|
+
T.any(
|
|
1553
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Perform,
|
|
1554
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Scroll,
|
|
1555
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Wait,
|
|
1556
|
+
ContextDev::WebScrapeParams::SharedParams::Action::WaitFor
|
|
1557
|
+
)
|
|
1558
|
+
end
|
|
1559
|
+
|
|
1560
|
+
class Perform < ContextDev::Internal::Type::BaseModel
|
|
1561
|
+
OrHash =
|
|
1562
|
+
T.type_alias do
|
|
1563
|
+
T.any(
|
|
1564
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Perform,
|
|
1565
|
+
ContextDev::Internal::AnyHash
|
|
1566
|
+
)
|
|
1567
|
+
end
|
|
1568
|
+
|
|
1569
|
+
sig { returns(String) }
|
|
1570
|
+
attr_accessor :action
|
|
1571
|
+
|
|
1572
|
+
sig { returns(Symbol) }
|
|
1573
|
+
attr_accessor :type
|
|
1574
|
+
|
|
1575
|
+
sig do
|
|
1576
|
+
params(action: String, type: Symbol).returns(T.attached_class)
|
|
1577
|
+
end
|
|
1578
|
+
def self.new(action:, type: :perform)
|
|
1579
|
+
end
|
|
1580
|
+
|
|
1581
|
+
sig { override.returns({ action: String, type: Symbol }) }
|
|
1582
|
+
def to_hash
|
|
1583
|
+
end
|
|
1584
|
+
end
|
|
1585
|
+
|
|
1586
|
+
class Scroll < ContextDev::Internal::Type::BaseModel
|
|
1587
|
+
OrHash =
|
|
1588
|
+
T.type_alias do
|
|
1589
|
+
T.any(
|
|
1590
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Scroll,
|
|
1591
|
+
ContextDev::Internal::AnyHash
|
|
1592
|
+
)
|
|
1593
|
+
end
|
|
1594
|
+
|
|
1595
|
+
sig { returns(Symbol) }
|
|
1596
|
+
attr_accessor :type
|
|
1597
|
+
|
|
1598
|
+
sig do
|
|
1599
|
+
returns(
|
|
1600
|
+
T.nilable(
|
|
1601
|
+
T.any(
|
|
1602
|
+
Integer,
|
|
1603
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Scroll::Amount::OrSymbol
|
|
1604
|
+
)
|
|
1605
|
+
)
|
|
1606
|
+
)
|
|
1607
|
+
end
|
|
1608
|
+
attr_reader :amount
|
|
1609
|
+
|
|
1610
|
+
sig do
|
|
1611
|
+
params(
|
|
1612
|
+
amount:
|
|
1613
|
+
T.any(
|
|
1614
|
+
Integer,
|
|
1615
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Scroll::Amount::OrSymbol
|
|
1616
|
+
)
|
|
1617
|
+
).void
|
|
1618
|
+
end
|
|
1619
|
+
attr_writer :amount
|
|
1620
|
+
|
|
1621
|
+
sig do
|
|
1622
|
+
returns(
|
|
1623
|
+
T.nilable(
|
|
1624
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Scroll::Direction::OrSymbol
|
|
1625
|
+
)
|
|
1626
|
+
)
|
|
1627
|
+
end
|
|
1628
|
+
attr_reader :direction
|
|
1629
|
+
|
|
1630
|
+
sig do
|
|
1631
|
+
params(
|
|
1632
|
+
direction:
|
|
1633
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Scroll::Direction::OrSymbol
|
|
1634
|
+
).void
|
|
1635
|
+
end
|
|
1636
|
+
attr_writer :direction
|
|
1637
|
+
|
|
1638
|
+
sig { returns(T.nilable(Integer)) }
|
|
1639
|
+
attr_reader :max_scrolls
|
|
1640
|
+
|
|
1641
|
+
sig { params(max_scrolls: Integer).void }
|
|
1642
|
+
attr_writer :max_scrolls
|
|
1643
|
+
|
|
1644
|
+
# Scroll this container. Omit to scroll the page.
|
|
1645
|
+
sig { returns(T.nilable(String)) }
|
|
1646
|
+
attr_reader :selector
|
|
1647
|
+
|
|
1648
|
+
sig { params(selector: String).void }
|
|
1649
|
+
attr_writer :selector
|
|
1650
|
+
|
|
1651
|
+
sig do
|
|
1652
|
+
params(
|
|
1653
|
+
amount:
|
|
1654
|
+
T.any(
|
|
1655
|
+
Integer,
|
|
1656
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Scroll::Amount::OrSymbol
|
|
1657
|
+
),
|
|
1658
|
+
direction:
|
|
1659
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Scroll::Direction::OrSymbol,
|
|
1660
|
+
max_scrolls: Integer,
|
|
1661
|
+
selector: String,
|
|
1662
|
+
type: Symbol
|
|
1663
|
+
).returns(T.attached_class)
|
|
1664
|
+
end
|
|
1665
|
+
def self.new(
|
|
1666
|
+
amount: nil,
|
|
1667
|
+
direction: nil,
|
|
1668
|
+
max_scrolls: nil,
|
|
1669
|
+
# Scroll this container. Omit to scroll the page.
|
|
1670
|
+
selector: nil,
|
|
1671
|
+
type: :scroll
|
|
1672
|
+
)
|
|
1673
|
+
end
|
|
1674
|
+
|
|
1675
|
+
sig do
|
|
1676
|
+
override.returns(
|
|
1677
|
+
{
|
|
1678
|
+
type: Symbol,
|
|
1679
|
+
amount:
|
|
1680
|
+
T.any(
|
|
1681
|
+
Integer,
|
|
1682
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Scroll::Amount::OrSymbol
|
|
1683
|
+
),
|
|
1684
|
+
direction:
|
|
1685
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Scroll::Direction::OrSymbol,
|
|
1686
|
+
max_scrolls: Integer,
|
|
1687
|
+
selector: String
|
|
1688
|
+
}
|
|
1689
|
+
)
|
|
1690
|
+
end
|
|
1691
|
+
def to_hash
|
|
1692
|
+
end
|
|
1693
|
+
|
|
1694
|
+
module Amount
|
|
1695
|
+
extend ContextDev::Internal::Type::Union
|
|
1696
|
+
|
|
1697
|
+
Variants =
|
|
1698
|
+
T.type_alias do
|
|
1699
|
+
T.any(
|
|
1700
|
+
Integer,
|
|
1701
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Scroll::Amount::TaggedSymbol
|
|
1702
|
+
)
|
|
1703
|
+
end
|
|
1704
|
+
|
|
1705
|
+
sig do
|
|
1706
|
+
override.returns(
|
|
1707
|
+
T::Array[
|
|
1708
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Scroll::Amount::Variants
|
|
1709
|
+
]
|
|
1710
|
+
)
|
|
1711
|
+
end
|
|
1712
|
+
def self.variants
|
|
1713
|
+
end
|
|
1714
|
+
|
|
1715
|
+
TaggedSymbol =
|
|
1716
|
+
T.type_alias do
|
|
1717
|
+
T.all(
|
|
1718
|
+
Symbol,
|
|
1719
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Scroll::Amount
|
|
1720
|
+
)
|
|
1721
|
+
end
|
|
1722
|
+
OrSymbol = T.type_alias { T.any(Symbol, String) }
|
|
1723
|
+
|
|
1724
|
+
VIEWPORT =
|
|
1725
|
+
T.let(
|
|
1726
|
+
:viewport,
|
|
1727
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Scroll::Amount::TaggedSymbol
|
|
1728
|
+
)
|
|
1729
|
+
MAX =
|
|
1730
|
+
T.let(
|
|
1731
|
+
:max,
|
|
1732
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Scroll::Amount::TaggedSymbol
|
|
1733
|
+
)
|
|
1734
|
+
end
|
|
1735
|
+
|
|
1736
|
+
module Direction
|
|
1737
|
+
extend ContextDev::Internal::Type::Enum
|
|
1738
|
+
|
|
1739
|
+
TaggedSymbol =
|
|
1740
|
+
T.type_alias do
|
|
1741
|
+
T.all(
|
|
1742
|
+
Symbol,
|
|
1743
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Scroll::Direction
|
|
1744
|
+
)
|
|
1745
|
+
end
|
|
1746
|
+
OrSymbol = T.type_alias { T.any(Symbol, String) }
|
|
1747
|
+
|
|
1748
|
+
DOWN =
|
|
1749
|
+
T.let(
|
|
1750
|
+
:down,
|
|
1751
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Scroll::Direction::TaggedSymbol
|
|
1752
|
+
)
|
|
1753
|
+
UP =
|
|
1754
|
+
T.let(
|
|
1755
|
+
:up,
|
|
1756
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Scroll::Direction::TaggedSymbol
|
|
1757
|
+
)
|
|
1758
|
+
LEFT =
|
|
1759
|
+
T.let(
|
|
1760
|
+
:left,
|
|
1761
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Scroll::Direction::TaggedSymbol
|
|
1762
|
+
)
|
|
1763
|
+
RIGHT =
|
|
1764
|
+
T.let(
|
|
1765
|
+
:right,
|
|
1766
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Scroll::Direction::TaggedSymbol
|
|
1767
|
+
)
|
|
1768
|
+
|
|
1769
|
+
sig do
|
|
1770
|
+
override.returns(
|
|
1771
|
+
T::Array[
|
|
1772
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Scroll::Direction::TaggedSymbol
|
|
1773
|
+
]
|
|
1774
|
+
)
|
|
1775
|
+
end
|
|
1776
|
+
def self.values
|
|
1777
|
+
end
|
|
1778
|
+
end
|
|
1779
|
+
end
|
|
1780
|
+
|
|
1781
|
+
class Wait < ContextDev::Internal::Type::BaseModel
|
|
1782
|
+
OrHash =
|
|
1783
|
+
T.type_alias do
|
|
1784
|
+
T.any(
|
|
1785
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Wait,
|
|
1786
|
+
ContextDev::Internal::AnyHash
|
|
1787
|
+
)
|
|
1788
|
+
end
|
|
1789
|
+
|
|
1790
|
+
sig { returns(Integer) }
|
|
1791
|
+
attr_accessor :milliseconds
|
|
1792
|
+
|
|
1793
|
+
sig { returns(Symbol) }
|
|
1794
|
+
attr_accessor :type
|
|
1795
|
+
|
|
1796
|
+
sig do
|
|
1797
|
+
params(milliseconds: Integer, type: Symbol).returns(
|
|
1798
|
+
T.attached_class
|
|
1799
|
+
)
|
|
1800
|
+
end
|
|
1801
|
+
def self.new(milliseconds:, type: :wait)
|
|
1802
|
+
end
|
|
1803
|
+
|
|
1804
|
+
sig { override.returns({ milliseconds: Integer, type: Symbol }) }
|
|
1805
|
+
def to_hash
|
|
1806
|
+
end
|
|
1807
|
+
end
|
|
1808
|
+
|
|
1809
|
+
class WaitFor < ContextDev::Internal::Type::BaseModel
|
|
1810
|
+
OrHash =
|
|
1811
|
+
T.type_alias do
|
|
1812
|
+
T.any(
|
|
1813
|
+
ContextDev::WebScrapeParams::SharedParams::Action::WaitFor,
|
|
1814
|
+
ContextDev::Internal::AnyHash
|
|
1815
|
+
)
|
|
1816
|
+
end
|
|
1817
|
+
|
|
1818
|
+
sig { returns(String) }
|
|
1819
|
+
attr_accessor :selector
|
|
1820
|
+
|
|
1821
|
+
sig { returns(Symbol) }
|
|
1822
|
+
attr_accessor :type
|
|
1823
|
+
|
|
1824
|
+
sig do
|
|
1825
|
+
params(selector: String, type: Symbol).returns(T.attached_class)
|
|
1826
|
+
end
|
|
1827
|
+
def self.new(selector:, type: :waitFor)
|
|
1828
|
+
end
|
|
1829
|
+
|
|
1830
|
+
sig { override.returns({ selector: String, type: Symbol }) }
|
|
1831
|
+
def to_hash
|
|
1832
|
+
end
|
|
1833
|
+
end
|
|
1834
|
+
|
|
1835
|
+
sig do
|
|
1836
|
+
override.returns(
|
|
1837
|
+
T::Array[
|
|
1838
|
+
ContextDev::WebScrapeParams::SharedParams::Action::Variants
|
|
1839
|
+
]
|
|
1840
|
+
)
|
|
1841
|
+
end
|
|
1842
|
+
def self.variants
|
|
1843
|
+
end
|
|
1844
|
+
end
|
|
1845
|
+
|
|
1846
|
+
class Parsers < ContextDev::Internal::Type::BaseModel
|
|
1847
|
+
OrHash =
|
|
1848
|
+
T.type_alias do
|
|
1849
|
+
T.any(
|
|
1850
|
+
ContextDev::WebScrapeParams::SharedParams::Parsers,
|
|
1851
|
+
ContextDev::Internal::AnyHash
|
|
1852
|
+
)
|
|
1853
|
+
end
|
|
1854
|
+
|
|
1855
|
+
# PDF text options for HTML, Markdown, and parsed fields.
|
|
1856
|
+
sig do
|
|
1857
|
+
returns(
|
|
1858
|
+
T.nilable(ContextDev::WebScrapeParams::SharedParams::Parsers::Pdf)
|
|
1859
|
+
)
|
|
1860
|
+
end
|
|
1861
|
+
attr_reader :pdf
|
|
1862
|
+
|
|
1863
|
+
sig do
|
|
1864
|
+
params(
|
|
1865
|
+
pdf:
|
|
1866
|
+
ContextDev::WebScrapeParams::SharedParams::Parsers::Pdf::OrHash
|
|
1867
|
+
).void
|
|
1868
|
+
end
|
|
1869
|
+
attr_writer :pdf
|
|
1870
|
+
|
|
1871
|
+
# Document parsing options.
|
|
1872
|
+
sig do
|
|
1873
|
+
params(
|
|
1874
|
+
pdf:
|
|
1875
|
+
ContextDev::WebScrapeParams::SharedParams::Parsers::Pdf::OrHash
|
|
1876
|
+
).returns(T.attached_class)
|
|
1877
|
+
end
|
|
1878
|
+
def self.new(
|
|
1879
|
+
# PDF text options for HTML, Markdown, and parsed fields.
|
|
1880
|
+
pdf: nil
|
|
1881
|
+
)
|
|
1882
|
+
end
|
|
1883
|
+
|
|
1884
|
+
sig do
|
|
1885
|
+
override.returns(
|
|
1886
|
+
{ pdf: ContextDev::WebScrapeParams::SharedParams::Parsers::Pdf }
|
|
1887
|
+
)
|
|
1888
|
+
end
|
|
1889
|
+
def to_hash
|
|
1890
|
+
end
|
|
1891
|
+
|
|
1892
|
+
class Pdf < ContextDev::Internal::Type::BaseModel
|
|
1893
|
+
OrHash =
|
|
1894
|
+
T.type_alias do
|
|
1895
|
+
T.any(
|
|
1896
|
+
ContextDev::WebScrapeParams::SharedParams::Parsers::Pdf,
|
|
1897
|
+
ContextDev::Internal::AnyHash
|
|
1898
|
+
)
|
|
1899
|
+
end
|
|
1900
|
+
|
|
1901
|
+
# Last page to parse. Must be at least startPage.
|
|
1902
|
+
sig { returns(T.nilable(Integer)) }
|
|
1903
|
+
attr_reader :end_page
|
|
1904
|
+
|
|
1905
|
+
sig { params(end_page: Integer).void }
|
|
1906
|
+
attr_writer :end_page
|
|
1907
|
+
|
|
1908
|
+
# Read text from scanned pages.
|
|
1909
|
+
sig do
|
|
1910
|
+
returns(
|
|
1911
|
+
T.nilable(
|
|
1912
|
+
ContextDev::WebScrapeParams::SharedParams::Parsers::Pdf::Ocr::OrSymbol
|
|
1913
|
+
)
|
|
1914
|
+
)
|
|
1915
|
+
end
|
|
1916
|
+
attr_reader :ocr
|
|
1917
|
+
|
|
1918
|
+
sig do
|
|
1919
|
+
params(
|
|
1920
|
+
ocr:
|
|
1921
|
+
ContextDev::WebScrapeParams::SharedParams::Parsers::Pdf::Ocr::OrSymbol
|
|
1922
|
+
).void
|
|
1923
|
+
end
|
|
1924
|
+
attr_writer :ocr
|
|
1925
|
+
|
|
1926
|
+
# First page to parse, starting at 1.
|
|
1927
|
+
sig { returns(T.nilable(Integer)) }
|
|
1928
|
+
attr_reader :start_page
|
|
1929
|
+
|
|
1930
|
+
sig { params(start_page: Integer).void }
|
|
1931
|
+
attr_writer :start_page
|
|
1932
|
+
|
|
1933
|
+
# PDF text options for HTML, Markdown, and parsed fields.
|
|
1934
|
+
sig do
|
|
1935
|
+
params(
|
|
1936
|
+
end_page: Integer,
|
|
1937
|
+
ocr:
|
|
1938
|
+
ContextDev::WebScrapeParams::SharedParams::Parsers::Pdf::Ocr::OrSymbol,
|
|
1939
|
+
start_page: Integer
|
|
1940
|
+
).returns(T.attached_class)
|
|
1941
|
+
end
|
|
1942
|
+
def self.new(
|
|
1943
|
+
# Last page to parse. Must be at least startPage.
|
|
1944
|
+
end_page: nil,
|
|
1945
|
+
# Read text from scanned pages.
|
|
1946
|
+
ocr: nil,
|
|
1947
|
+
# First page to parse, starting at 1.
|
|
1948
|
+
start_page: nil
|
|
1949
|
+
)
|
|
1950
|
+
end
|
|
1951
|
+
|
|
1952
|
+
sig do
|
|
1953
|
+
override.returns(
|
|
1954
|
+
{
|
|
1955
|
+
end_page: Integer,
|
|
1956
|
+
ocr:
|
|
1957
|
+
ContextDev::WebScrapeParams::SharedParams::Parsers::Pdf::Ocr::OrSymbol,
|
|
1958
|
+
start_page: Integer
|
|
1959
|
+
}
|
|
1960
|
+
)
|
|
1961
|
+
end
|
|
1962
|
+
def to_hash
|
|
1963
|
+
end
|
|
1964
|
+
|
|
1965
|
+
# Read text from scanned pages.
|
|
1966
|
+
module Ocr
|
|
1967
|
+
extend ContextDev::Internal::Type::Enum
|
|
1968
|
+
|
|
1969
|
+
TaggedSymbol =
|
|
1970
|
+
T.type_alias do
|
|
1971
|
+
T.all(
|
|
1972
|
+
Symbol,
|
|
1973
|
+
ContextDev::WebScrapeParams::SharedParams::Parsers::Pdf::Ocr
|
|
1974
|
+
)
|
|
1975
|
+
end
|
|
1976
|
+
OrSymbol = T.type_alias { T.any(Symbol, String) }
|
|
1977
|
+
|
|
1978
|
+
OFF =
|
|
1979
|
+
T.let(
|
|
1980
|
+
:off,
|
|
1981
|
+
ContextDev::WebScrapeParams::SharedParams::Parsers::Pdf::Ocr::TaggedSymbol
|
|
1982
|
+
)
|
|
1983
|
+
AUTO =
|
|
1984
|
+
T.let(
|
|
1985
|
+
:auto,
|
|
1986
|
+
ContextDev::WebScrapeParams::SharedParams::Parsers::Pdf::Ocr::TaggedSymbol
|
|
1987
|
+
)
|
|
1988
|
+
|
|
1989
|
+
sig do
|
|
1990
|
+
override.returns(
|
|
1991
|
+
T::Array[
|
|
1992
|
+
ContextDev::WebScrapeParams::SharedParams::Parsers::Pdf::Ocr::TaggedSymbol
|
|
1993
|
+
]
|
|
1994
|
+
)
|
|
1995
|
+
end
|
|
1996
|
+
def self.values
|
|
1997
|
+
end
|
|
1998
|
+
end
|
|
1999
|
+
end
|
|
2000
|
+
end
|
|
2001
|
+
|
|
2002
|
+
# Override the browser color scheme.
|
|
2003
|
+
module Theme
|
|
2004
|
+
extend ContextDev::Internal::Type::Enum
|
|
2005
|
+
|
|
2006
|
+
TaggedSymbol =
|
|
2007
|
+
T.type_alias do
|
|
2008
|
+
T.all(Symbol, ContextDev::WebScrapeParams::SharedParams::Theme)
|
|
2009
|
+
end
|
|
2010
|
+
OrSymbol = T.type_alias { T.any(Symbol, String) }
|
|
2011
|
+
|
|
2012
|
+
LIGHT =
|
|
2013
|
+
T.let(
|
|
2014
|
+
:light,
|
|
2015
|
+
ContextDev::WebScrapeParams::SharedParams::Theme::TaggedSymbol
|
|
2016
|
+
)
|
|
2017
|
+
DARK =
|
|
2018
|
+
T.let(
|
|
2019
|
+
:dark,
|
|
2020
|
+
ContextDev::WebScrapeParams::SharedParams::Theme::TaggedSymbol
|
|
2021
|
+
)
|
|
2022
|
+
|
|
2023
|
+
sig do
|
|
2024
|
+
override.returns(
|
|
2025
|
+
T::Array[
|
|
2026
|
+
ContextDev::WebScrapeParams::SharedParams::Theme::TaggedSymbol
|
|
2027
|
+
]
|
|
2028
|
+
)
|
|
2029
|
+
end
|
|
2030
|
+
def self.values
|
|
2031
|
+
end
|
|
2032
|
+
end
|
|
2033
|
+
|
|
2034
|
+
class Viewport < ContextDev::Internal::Type::BaseModel
|
|
2035
|
+
OrHash =
|
|
2036
|
+
T.type_alias do
|
|
2037
|
+
T.any(
|
|
2038
|
+
ContextDev::WebScrapeParams::SharedParams::Viewport,
|
|
2039
|
+
ContextDev::Internal::AnyHash
|
|
2040
|
+
)
|
|
2041
|
+
end
|
|
2042
|
+
|
|
2043
|
+
sig { returns(T.nilable(Integer)) }
|
|
2044
|
+
attr_reader :height
|
|
2045
|
+
|
|
2046
|
+
sig { params(height: Integer).void }
|
|
2047
|
+
attr_writer :height
|
|
2048
|
+
|
|
2049
|
+
sig { returns(T.nilable(Integer)) }
|
|
2050
|
+
attr_reader :width
|
|
2051
|
+
|
|
2052
|
+
sig { params(width: Integer).void }
|
|
2053
|
+
attr_writer :width
|
|
2054
|
+
|
|
2055
|
+
# Browser dimensions in pixels.
|
|
2056
|
+
sig do
|
|
2057
|
+
params(height: Integer, width: Integer).returns(T.attached_class)
|
|
2058
|
+
end
|
|
2059
|
+
def self.new(height: nil, width: nil)
|
|
2060
|
+
end
|
|
2061
|
+
|
|
2062
|
+
sig { override.returns({ height: Integer, width: Integer }) }
|
|
2063
|
+
def to_hash
|
|
2064
|
+
end
|
|
2065
|
+
end
|
|
2066
|
+
|
|
2067
|
+
# After actions, wait this many milliseconds or until a CSS selector is visible.
|
|
2068
|
+
# Defaults to 500 ms, or 2000 ms with frames or an XML URL. Set 0 to skip.
|
|
2069
|
+
module WaitFor
|
|
2070
|
+
extend ContextDev::Internal::Type::Union
|
|
2071
|
+
|
|
2072
|
+
Variants = T.type_alias { T.any(Integer, String) }
|
|
2073
|
+
|
|
2074
|
+
sig do
|
|
2075
|
+
override.returns(
|
|
2076
|
+
T::Array[
|
|
2077
|
+
ContextDev::WebScrapeParams::SharedParams::WaitFor::Variants
|
|
2078
|
+
]
|
|
2079
|
+
)
|
|
2080
|
+
end
|
|
2081
|
+
def self.variants
|
|
2082
|
+
end
|
|
2083
|
+
end
|
|
2084
|
+
end
|
|
2085
|
+
|
|
2086
|
+
class TimeoutOpts < ContextDev::Internal::Type::BaseModel
|
|
2087
|
+
OrHash =
|
|
2088
|
+
T.type_alias do
|
|
2089
|
+
T.any(
|
|
2090
|
+
ContextDev::WebScrapeParams::TimeoutOpts,
|
|
2091
|
+
ContextDev::Internal::AnyHash
|
|
2092
|
+
)
|
|
2093
|
+
end
|
|
2094
|
+
|
|
2095
|
+
# Request deadline in milliseconds. Maximum: 300000 (5 minutes).
|
|
2096
|
+
sig { returns(Integer) }
|
|
2097
|
+
attr_accessor :milliseconds
|
|
2098
|
+
|
|
2099
|
+
# What to do at the deadline. "fail" returns 408 REQUEST_TIMEOUT without charging
|
|
2100
|
+
# credits. "return-partial" returns usable results collected so far; if none are
|
|
2101
|
+
# available, the request still fails without charging credits. Partial results are
|
|
2102
|
+
# not cached as complete results. "return-partial" requires milliseconds of at
|
|
2103
|
+
# least 5000.
|
|
2104
|
+
sig do
|
|
2105
|
+
returns(
|
|
2106
|
+
T.nilable(
|
|
2107
|
+
ContextDev::WebScrapeParams::TimeoutOpts::Behavior::OrSymbol
|
|
2108
|
+
)
|
|
2109
|
+
)
|
|
2110
|
+
end
|
|
2111
|
+
attr_reader :behavior
|
|
2112
|
+
|
|
2113
|
+
sig do
|
|
2114
|
+
params(
|
|
2115
|
+
behavior:
|
|
2116
|
+
ContextDev::WebScrapeParams::TimeoutOpts::Behavior::OrSymbol
|
|
2117
|
+
).void
|
|
2118
|
+
end
|
|
2119
|
+
attr_writer :behavior
|
|
2120
|
+
|
|
2121
|
+
# Total deadline, including navigation, actions, waiting, and all outputs.
|
|
2122
|
+
# Defaults to 60000 milliseconds with behavior fail. Use return-partial to capture
|
|
2123
|
+
# the current page state and return captured images if image processing cannot
|
|
2124
|
+
# finish before the deadline; these responses set isPartial and are not cached.
|
|
2125
|
+
# Every requested format must still be available. Fixed waits must fit before a
|
|
2126
|
+
# response reserve of up to 5000 milliseconds (at most one quarter of the timeout)
|
|
2127
|
+
# when using return-partial.
|
|
2128
|
+
sig do
|
|
2129
|
+
params(
|
|
2130
|
+
milliseconds: Integer,
|
|
2131
|
+
behavior:
|
|
2132
|
+
ContextDev::WebScrapeParams::TimeoutOpts::Behavior::OrSymbol
|
|
2133
|
+
).returns(T.attached_class)
|
|
2134
|
+
end
|
|
2135
|
+
def self.new(
|
|
2136
|
+
# Request deadline in milliseconds. Maximum: 300000 (5 minutes).
|
|
2137
|
+
milliseconds:,
|
|
2138
|
+
# What to do at the deadline. "fail" returns 408 REQUEST_TIMEOUT without charging
|
|
2139
|
+
# credits. "return-partial" returns usable results collected so far; if none are
|
|
2140
|
+
# available, the request still fails without charging credits. Partial results are
|
|
2141
|
+
# not cached as complete results. "return-partial" requires milliseconds of at
|
|
2142
|
+
# least 5000.
|
|
2143
|
+
behavior: nil
|
|
2144
|
+
)
|
|
2145
|
+
end
|
|
2146
|
+
|
|
2147
|
+
sig do
|
|
2148
|
+
override.returns(
|
|
2149
|
+
{
|
|
2150
|
+
milliseconds: Integer,
|
|
2151
|
+
behavior:
|
|
2152
|
+
ContextDev::WebScrapeParams::TimeoutOpts::Behavior::OrSymbol
|
|
2153
|
+
}
|
|
2154
|
+
)
|
|
2155
|
+
end
|
|
2156
|
+
def to_hash
|
|
2157
|
+
end
|
|
2158
|
+
|
|
2159
|
+
# What to do at the deadline. "fail" returns 408 REQUEST_TIMEOUT without charging
|
|
2160
|
+
# credits. "return-partial" returns usable results collected so far; if none are
|
|
2161
|
+
# available, the request still fails without charging credits. Partial results are
|
|
2162
|
+
# not cached as complete results. "return-partial" requires milliseconds of at
|
|
2163
|
+
# least 5000.
|
|
2164
|
+
module Behavior
|
|
2165
|
+
extend ContextDev::Internal::Type::Enum
|
|
2166
|
+
|
|
2167
|
+
TaggedSymbol =
|
|
2168
|
+
T.type_alias do
|
|
2169
|
+
T.all(Symbol, ContextDev::WebScrapeParams::TimeoutOpts::Behavior)
|
|
2170
|
+
end
|
|
2171
|
+
OrSymbol = T.type_alias { T.any(Symbol, String) }
|
|
2172
|
+
|
|
2173
|
+
FAIL =
|
|
2174
|
+
T.let(
|
|
2175
|
+
:fail,
|
|
2176
|
+
ContextDev::WebScrapeParams::TimeoutOpts::Behavior::TaggedSymbol
|
|
2177
|
+
)
|
|
2178
|
+
RETURN_PARTIAL =
|
|
2179
|
+
T.let(
|
|
2180
|
+
:"return-partial",
|
|
2181
|
+
ContextDev::WebScrapeParams::TimeoutOpts::Behavior::TaggedSymbol
|
|
2182
|
+
)
|
|
2183
|
+
|
|
2184
|
+
sig do
|
|
2185
|
+
override.returns(
|
|
2186
|
+
T::Array[
|
|
2187
|
+
ContextDev::WebScrapeParams::TimeoutOpts::Behavior::TaggedSymbol
|
|
2188
|
+
]
|
|
2189
|
+
)
|
|
2190
|
+
end
|
|
2191
|
+
def self.values
|
|
2192
|
+
end
|
|
2193
|
+
end
|
|
2194
|
+
end
|
|
2195
|
+
|
|
2196
|
+
# Zero data retention. Bypasses caches and uploads; excludes request/response
|
|
2197
|
+
# content and tags from logs. Must be enabled for your organization. Not available
|
|
2198
|
+
# with the highlights output.
|
|
2199
|
+
module Zdr
|
|
2200
|
+
extend ContextDev::Internal::Type::Enum
|
|
2201
|
+
|
|
2202
|
+
TaggedSymbol =
|
|
2203
|
+
T.type_alias { T.all(Symbol, ContextDev::WebScrapeParams::Zdr) }
|
|
2204
|
+
OrSymbol = T.type_alias { T.any(Symbol, String) }
|
|
2205
|
+
|
|
2206
|
+
ENABLED =
|
|
2207
|
+
T.let(:enabled, ContextDev::WebScrapeParams::Zdr::TaggedSymbol)
|
|
2208
|
+
DISABLED =
|
|
2209
|
+
T.let(:disabled, ContextDev::WebScrapeParams::Zdr::TaggedSymbol)
|
|
2210
|
+
|
|
2211
|
+
sig do
|
|
2212
|
+
override.returns(
|
|
2213
|
+
T::Array[ContextDev::WebScrapeParams::Zdr::TaggedSymbol]
|
|
2214
|
+
)
|
|
2215
|
+
end
|
|
2216
|
+
def self.values
|
|
2217
|
+
end
|
|
2218
|
+
end
|
|
2219
|
+
end
|
|
2220
|
+
end
|
|
2221
|
+
end
|