context.dev 2.10.0 → 2.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +21 -0
- data/README.md +1 -1
- data/lib/context_dev/models/ai_extract_product_response.rb +48 -1
- data/lib/context_dev/models/ai_extract_products_response.rb +48 -1
- data/lib/context_dev/models/brand_retrieve_response.rb +29 -1
- data/lib/context_dev/models/brand_retrieve_simplified_response.rb +30 -1
- data/lib/context_dev/models/news_search_params.rb +1 -0
- data/lib/context_dev/models/news_search_response.rb +68 -22
- data/lib/context_dev/models/web_extract_params.rb +169 -6
- data/lib/context_dev/models/web_extract_response.rb +85 -1
- data/lib/context_dev/models/web_screenshot_params.rb +13 -1
- data/lib/context_dev/models/web_web_crawl_md_params.rb +7 -5
- data/lib/context_dev/models/web_web_scrape_html_params.rb +99 -3
- data/lib/context_dev/models/web_web_scrape_images_params.rb +99 -3
- data/lib/context_dev/models/web_web_scrape_images_response.rb +85 -1
- data/lib/context_dev/models/web_web_scrape_md_params.rb +99 -3
- data/lib/context_dev/resources/web.rb +13 -8
- data/lib/context_dev/version.rb +1 -1
- data/rbi/context_dev/models/ai_extract_product_response.rbi +99 -0
- data/rbi/context_dev/models/ai_extract_products_response.rbi +99 -0
- data/rbi/context_dev/models/brand_retrieve_response.rbi +80 -3
- data/rbi/context_dev/models/brand_retrieve_simplified_response.rbi +80 -3
- data/rbi/context_dev/models/news_search_params.rbi +5 -0
- data/rbi/context_dev/models/news_search_response.rbi +70 -4
- data/rbi/context_dev/models/web_extract_params.rbi +353 -8
- data/rbi/context_dev/models/web_extract_response.rbi +172 -3
- data/rbi/context_dev/models/web_screenshot_params.rbi +19 -0
- data/rbi/context_dev/models/web_web_crawl_md_params.rbi +10 -6
- data/rbi/context_dev/models/web_web_scrape_html_params.rbi +218 -4
- data/rbi/context_dev/models/web_web_scrape_images_params.rbi +218 -4
- data/rbi/context_dev/models/web_web_scrape_images_response.rbi +169 -0
- data/rbi/context_dev/models/web_web_scrape_md_params.rbi +218 -4
- data/rbi/context_dev/resources/web.rbi +38 -10
- data/sig/context_dev/models/ai_extract_product_response.rbs +40 -0
- data/sig/context_dev/models/ai_extract_products_response.rbs +40 -0
- data/sig/context_dev/models/brand_retrieve_response.rbs +33 -3
- data/sig/context_dev/models/brand_retrieve_simplified_response.rbs +33 -3
- data/sig/context_dev/models/news_search_params.rbs +2 -0
- data/sig/context_dev/models/web_extract_params.rbs +117 -0
- data/sig/context_dev/models/web_extract_response.rbs +81 -3
- data/sig/context_dev/models/web_screenshot_params.rbs +7 -0
- data/sig/context_dev/models/web_web_scrape_html_params.rbs +74 -0
- data/sig/context_dev/models/web_web_scrape_images_params.rbs +74 -0
- data/sig/context_dev/models/web_web_scrape_images_response.rbs +78 -0
- data/sig/context_dev/models/web_web_scrape_md_params.rbs +74 -0
- data/sig/context_dev/resources/web.rbs +2 -0
- metadata +2 -2
|
@@ -8,9 +8,11 @@ module ContextDev
|
|
|
8
8
|
include ContextDev::Internal::Type::RequestParameters
|
|
9
9
|
|
|
10
10
|
# @!attribute schema
|
|
11
|
-
# JSON Schema for the returned data object.
|
|
12
|
-
#
|
|
13
|
-
#
|
|
11
|
+
# JSON Schema for the returned data object. Image fields such as `image_urls` or
|
|
12
|
+
# `product_photos` automatically make page image references available to
|
|
13
|
+
# extraction, so product data and photos can be returned in one call. TypeScript
|
|
14
|
+
# Zod users can pass a JSON Schema generated from a Zod object; Python users can
|
|
15
|
+
# pass the equivalent JSON Schema object.
|
|
14
16
|
#
|
|
15
17
|
# @return [Hash{Symbol=>Object}]
|
|
16
18
|
required :schema, ContextDev::Internal::Type::HashOf[ContextDev::Internal::Type::Unknown]
|
|
@@ -22,6 +24,15 @@ module ContextDev
|
|
|
22
24
|
# @return [String]
|
|
23
25
|
required :url, String
|
|
24
26
|
|
|
27
|
+
# @!attribute actions
|
|
28
|
+
# Optional browser actions executed in order on the requested page after it loads,
|
|
29
|
+
# before links are discovered or additional pages are crawled. Requires a paid
|
|
30
|
+
# plan. When actions are provided and stopAfterMs is omitted, the crawl budget
|
|
31
|
+
# defaults to 110000 ms.
|
|
32
|
+
#
|
|
33
|
+
# @return [Array<ContextDev::Models::WebExtractParams::Action::Wait, ContextDev::Models::WebExtractParams::Action::Perform, ContextDev::Models::WebExtractParams::Action::Scroll>, nil]
|
|
34
|
+
optional :actions, -> { ContextDev::Internal::Type::ArrayOf[union: ContextDev::WebExtractParams::Action] }
|
|
35
|
+
|
|
25
36
|
# @!attribute fact_check
|
|
26
37
|
# When true, every returned value must be grounded in facts stated on the page;
|
|
27
38
|
# fields that cannot be supported by the page are returned as null/empty. When
|
|
@@ -87,7 +98,8 @@ module ContextDev
|
|
|
87
98
|
|
|
88
99
|
# @!attribute stop_after_ms
|
|
89
100
|
# Soft time budget for the crawl in milliseconds. Min: 10000 (10s). Max: 110000
|
|
90
|
-
# (110s).
|
|
101
|
+
# (110s). Defaults to 80000 (80s), or 110000 (110s) when browser actions are
|
|
102
|
+
# provided.
|
|
91
103
|
#
|
|
92
104
|
# @return [Integer, nil]
|
|
93
105
|
optional :stop_after_ms, Integer, api_name: :stopAfterMs
|
|
@@ -113,14 +125,16 @@ module ContextDev
|
|
|
113
125
|
# @return [Integer, nil]
|
|
114
126
|
optional :wait_for_ms, Integer, api_name: :waitForMs
|
|
115
127
|
|
|
116
|
-
# @!method initialize(schema:, url:, fact_check: nil, follow_subdomains: nil, include_frames: nil, instructions: nil, max_age_ms: nil, max_depth: nil, max_pages: nil, pdf: nil, settle_animations: nil, stop_after_ms: nil, tags: nil, timeout_ms: nil, wait_for_ms: nil, request_options: {})
|
|
128
|
+
# @!method initialize(schema:, url:, actions: nil, fact_check: nil, follow_subdomains: nil, include_frames: nil, instructions: nil, max_age_ms: nil, max_depth: nil, max_pages: nil, pdf: nil, settle_animations: nil, stop_after_ms: nil, tags: nil, timeout_ms: nil, wait_for_ms: nil, request_options: {})
|
|
117
129
|
# Some parameter documentations has been truncated, see
|
|
118
130
|
# {ContextDev::Models::WebExtractParams} for more details.
|
|
119
131
|
#
|
|
120
|
-
# @param schema [Hash{Symbol=>Object}] JSON Schema for the returned data object.
|
|
132
|
+
# @param schema [Hash{Symbol=>Object}] JSON Schema for the returned data object. Image fields such as `image_urls` or `
|
|
121
133
|
#
|
|
122
134
|
# @param url [String] The starting website URL to crawl and extract from. Must include http:// or http
|
|
123
135
|
#
|
|
136
|
+
# @param actions [Array<ContextDev::Models::WebExtractParams::Action::Wait, ContextDev::Models::WebExtractParams::Action::Perform, ContextDev::Models::WebExtractParams::Action::Scroll>] Optional browser actions executed in order on the requested page after it loads,
|
|
137
|
+
#
|
|
124
138
|
# @param fact_check [Boolean] When true, every returned value must be grounded in facts stated on the page; fi
|
|
125
139
|
#
|
|
126
140
|
# @param follow_subdomains [Boolean] When true, follow links on subdomains of the starting URL's domain.
|
|
@@ -149,6 +163,155 @@ module ContextDev
|
|
|
149
163
|
#
|
|
150
164
|
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}]
|
|
151
165
|
|
|
166
|
+
# Browser action discriminated by `do`. Each variant exposes only its applicable
|
|
167
|
+
# fields.
|
|
168
|
+
module Action
|
|
169
|
+
extend ContextDev::Internal::Type::Union
|
|
170
|
+
|
|
171
|
+
discriminator :do
|
|
172
|
+
|
|
173
|
+
# Pause for a fixed number of milliseconds before continuing to the next action.
|
|
174
|
+
variant :wait, -> { ContextDev::WebExtractParams::Action::Wait }
|
|
175
|
+
|
|
176
|
+
# Resolve and perform one natural-language browser action.
|
|
177
|
+
variant :perform, -> { ContextDev::WebExtractParams::Action::Perform }
|
|
178
|
+
|
|
179
|
+
# Scroll the page or a selected scrollable container, waiting adaptively for content and dimensions to settle after each iteration.
|
|
180
|
+
variant :scroll, -> { ContextDev::WebExtractParams::Action::Scroll }
|
|
181
|
+
|
|
182
|
+
class Wait < ContextDev::Internal::Type::BaseModel
|
|
183
|
+
# @!attribute do_
|
|
184
|
+
#
|
|
185
|
+
# @return [Symbol, :wait]
|
|
186
|
+
required :do_, const: :wait, api_name: :do
|
|
187
|
+
|
|
188
|
+
# @!attribute time_ms
|
|
189
|
+
#
|
|
190
|
+
# @return [Integer]
|
|
191
|
+
required :time_ms, Integer, api_name: :timeMs
|
|
192
|
+
|
|
193
|
+
# @!method initialize(time_ms:, do_: :wait)
|
|
194
|
+
# Pause for a fixed number of milliseconds before continuing to the next action.
|
|
195
|
+
#
|
|
196
|
+
# @param time_ms [Integer]
|
|
197
|
+
# @param do_ [Symbol, :wait]
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
class Perform < ContextDev::Internal::Type::BaseModel
|
|
201
|
+
# @!attribute action
|
|
202
|
+
#
|
|
203
|
+
# @return [String]
|
|
204
|
+
required :action, String
|
|
205
|
+
|
|
206
|
+
# @!attribute do_
|
|
207
|
+
#
|
|
208
|
+
# @return [Symbol, :perform]
|
|
209
|
+
required :do_, const: :perform, api_name: :do
|
|
210
|
+
|
|
211
|
+
# @!method initialize(action:, do_: :perform)
|
|
212
|
+
# Resolve and perform one natural-language browser action.
|
|
213
|
+
#
|
|
214
|
+
# @param action [String]
|
|
215
|
+
# @param do_ [Symbol, :perform]
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
class Scroll < ContextDev::Internal::Type::BaseModel
|
|
219
|
+
# @!attribute do_
|
|
220
|
+
#
|
|
221
|
+
# @return [Symbol, :scroll]
|
|
222
|
+
required :do_, const: :scroll, api_name: :do
|
|
223
|
+
|
|
224
|
+
# @!attribute amount
|
|
225
|
+
# Pixels per scroll, one visible viewport, or the current scroll boundary.
|
|
226
|
+
# Defaults to viewport.
|
|
227
|
+
#
|
|
228
|
+
# @return [Integer, Symbol, ContextDev::Models::WebExtractParams::Action::Scroll::Amount, nil]
|
|
229
|
+
optional :amount, union: -> { ContextDev::WebExtractParams::Action::Scroll::Amount }
|
|
230
|
+
|
|
231
|
+
# @!attribute container
|
|
232
|
+
# CSS selector for the first matching scroll container. Defaults to the page.
|
|
233
|
+
#
|
|
234
|
+
# @return [String, nil]
|
|
235
|
+
optional :container, String
|
|
236
|
+
|
|
237
|
+
# @!attribute direction
|
|
238
|
+
# Direction to scroll. Defaults to down.
|
|
239
|
+
#
|
|
240
|
+
# @return [Symbol, ContextDev::Models::WebExtractParams::Action::Scroll::Direction, nil]
|
|
241
|
+
optional :direction, enum: -> { ContextDev::WebExtractParams::Action::Scroll::Direction }
|
|
242
|
+
|
|
243
|
+
# @!attribute max_scrolls
|
|
244
|
+
# Maximum scroll iterations. Stops early when scrolling and scrollable extent stop
|
|
245
|
+
# changing. Defaults to 1.
|
|
246
|
+
#
|
|
247
|
+
# @return [Integer, nil]
|
|
248
|
+
optional :max_scrolls, Integer, api_name: :maxScrolls
|
|
249
|
+
|
|
250
|
+
# @!method initialize(amount: nil, container: nil, direction: nil, max_scrolls: nil, do_: :scroll)
|
|
251
|
+
# Some parameter documentations has been truncated, see
|
|
252
|
+
# {ContextDev::Models::WebExtractParams::Action::Scroll} for more details.
|
|
253
|
+
#
|
|
254
|
+
# Scroll the page or a selected scrollable container, waiting adaptively for
|
|
255
|
+
# content and dimensions to settle after each iteration.
|
|
256
|
+
#
|
|
257
|
+
# @param amount [Integer, Symbol, ContextDev::Models::WebExtractParams::Action::Scroll::Amount] Pixels per scroll, one visible viewport, or the current scroll boundary. Default
|
|
258
|
+
#
|
|
259
|
+
# @param container [String] CSS selector for the first matching scroll container. Defaults to the page.
|
|
260
|
+
#
|
|
261
|
+
# @param direction [Symbol, ContextDev::Models::WebExtractParams::Action::Scroll::Direction] Direction to scroll. Defaults to down.
|
|
262
|
+
#
|
|
263
|
+
# @param max_scrolls [Integer] Maximum scroll iterations. Stops early when scrolling and scrollable extent stop
|
|
264
|
+
#
|
|
265
|
+
# @param do_ [Symbol, :scroll]
|
|
266
|
+
|
|
267
|
+
# Pixels per scroll, one visible viewport, or the current scroll boundary.
|
|
268
|
+
# Defaults to viewport.
|
|
269
|
+
#
|
|
270
|
+
# @see ContextDev::Models::WebExtractParams::Action::Scroll#amount
|
|
271
|
+
module Amount
|
|
272
|
+
extend ContextDev::Internal::Type::Union
|
|
273
|
+
|
|
274
|
+
variant Integer
|
|
275
|
+
|
|
276
|
+
variant const: -> { ContextDev::Models::WebExtractParams::Action::Scroll::Amount::VIEWPORT }
|
|
277
|
+
|
|
278
|
+
variant const: -> { ContextDev::Models::WebExtractParams::Action::Scroll::Amount::MAX }
|
|
279
|
+
|
|
280
|
+
# @!method self.variants
|
|
281
|
+
# @return [Array(Integer, Symbol)]
|
|
282
|
+
|
|
283
|
+
define_sorbet_constant!(:Variants) do
|
|
284
|
+
T.type_alias { T.any(Integer, ContextDev::WebExtractParams::Action::Scroll::Amount::TaggedSymbol) }
|
|
285
|
+
end
|
|
286
|
+
|
|
287
|
+
# @!group
|
|
288
|
+
|
|
289
|
+
VIEWPORT = :viewport
|
|
290
|
+
MAX = :max
|
|
291
|
+
|
|
292
|
+
# @!endgroup
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
# Direction to scroll. Defaults to down.
|
|
296
|
+
#
|
|
297
|
+
# @see ContextDev::Models::WebExtractParams::Action::Scroll#direction
|
|
298
|
+
module Direction
|
|
299
|
+
extend ContextDev::Internal::Type::Enum
|
|
300
|
+
|
|
301
|
+
UP = :up
|
|
302
|
+
DOWN = :down
|
|
303
|
+
LEFT = :left
|
|
304
|
+
RIGHT = :right
|
|
305
|
+
|
|
306
|
+
# @!method self.values
|
|
307
|
+
# @return [Array<Symbol>]
|
|
308
|
+
end
|
|
309
|
+
end
|
|
310
|
+
|
|
311
|
+
# @!method self.variants
|
|
312
|
+
# @return [Array(ContextDev::Models::WebExtractParams::Action::Wait, ContextDev::Models::WebExtractParams::Action::Perform, ContextDev::Models::WebExtractParams::Action::Scroll)]
|
|
313
|
+
end
|
|
314
|
+
|
|
152
315
|
class Pdf < ContextDev::Internal::Type::BaseModel
|
|
153
316
|
# @!attribute end_
|
|
154
317
|
# Last 1-based PDF page to parse. Must be greater than or equal to start when both
|
|
@@ -90,7 +90,15 @@ module ContextDev
|
|
|
90
90
|
# @return [Integer]
|
|
91
91
|
required :num_urls, Integer, api_name: :numUrls
|
|
92
92
|
|
|
93
|
-
# @!
|
|
93
|
+
# @!attribute actions_applied
|
|
94
|
+
# One verified outcome per requested browser action, in request order.
|
|
95
|
+
#
|
|
96
|
+
# @return [Array<ContextDev::Models::WebExtractResponse::Metadata::ActionsApplied>, nil]
|
|
97
|
+
optional :actions_applied,
|
|
98
|
+
-> { ContextDev::Internal::Type::ArrayOf[ContextDev::Models::WebExtractResponse::Metadata::ActionsApplied] },
|
|
99
|
+
api_name: :actionsApplied
|
|
100
|
+
|
|
101
|
+
# @!method initialize(max_crawl_depth:, num_blocked:, num_failed:, num_skipped:, num_succeeded:, num_urls:, actions_applied: nil)
|
|
94
102
|
# Some parameter documentations has been truncated, see
|
|
95
103
|
# {ContextDev::Models::WebExtractResponse::Metadata} for more details.
|
|
96
104
|
#
|
|
@@ -105,6 +113,82 @@ module ContextDev
|
|
|
105
113
|
# @param num_succeeded [Integer]
|
|
106
114
|
#
|
|
107
115
|
# @param num_urls [Integer]
|
|
116
|
+
#
|
|
117
|
+
# @param actions_applied [Array<ContextDev::Models::WebExtractResponse::Metadata::ActionsApplied>] One verified outcome per requested browser action, in request order.
|
|
118
|
+
|
|
119
|
+
class ActionsApplied < ContextDev::Internal::Type::BaseModel
|
|
120
|
+
# @!attribute instruction
|
|
121
|
+
#
|
|
122
|
+
# @return [String]
|
|
123
|
+
required :instruction, String
|
|
124
|
+
|
|
125
|
+
# @!attribute status
|
|
126
|
+
# Applied means the requested page state was visibly verified. Failed means it was
|
|
127
|
+
# not verified. Skipped means it was not attempted.
|
|
128
|
+
#
|
|
129
|
+
# @return [Symbol, ContextDev::Models::WebExtractResponse::Metadata::ActionsApplied::Status]
|
|
130
|
+
required :status, enum: -> { ContextDev::Models::WebExtractResponse::Metadata::ActionsApplied::Status }
|
|
131
|
+
|
|
132
|
+
# @!attribute completion_evidence
|
|
133
|
+
# Visible page evidence used to verify an applied action.
|
|
134
|
+
#
|
|
135
|
+
# @return [String, nil]
|
|
136
|
+
optional :completion_evidence, String, api_name: :completionEvidence
|
|
137
|
+
|
|
138
|
+
# @!attribute duration_ms
|
|
139
|
+
#
|
|
140
|
+
# @return [Float, nil]
|
|
141
|
+
optional :duration_ms, Float, api_name: :durationMs
|
|
142
|
+
|
|
143
|
+
# @!attribute error
|
|
144
|
+
#
|
|
145
|
+
# @return [String, nil]
|
|
146
|
+
optional :error, String
|
|
147
|
+
|
|
148
|
+
# @!attribute method_
|
|
149
|
+
#
|
|
150
|
+
# @return [String, nil]
|
|
151
|
+
optional :method_, String, api_name: :method
|
|
152
|
+
|
|
153
|
+
# @!attribute target_description
|
|
154
|
+
#
|
|
155
|
+
# @return [String, nil]
|
|
156
|
+
optional :target_description, String, api_name: :targetDescription
|
|
157
|
+
|
|
158
|
+
# @!method initialize(instruction:, status:, completion_evidence: nil, duration_ms: nil, error: nil, method_: nil, target_description: nil)
|
|
159
|
+
# Some parameter documentations has been truncated, see
|
|
160
|
+
# {ContextDev::Models::WebExtractResponse::Metadata::ActionsApplied} for more
|
|
161
|
+
# details.
|
|
162
|
+
#
|
|
163
|
+
# @param instruction [String]
|
|
164
|
+
#
|
|
165
|
+
# @param status [Symbol, ContextDev::Models::WebExtractResponse::Metadata::ActionsApplied::Status] Applied means the requested page state was visibly verified. Failed means it was
|
|
166
|
+
#
|
|
167
|
+
# @param completion_evidence [String] Visible page evidence used to verify an applied action.
|
|
168
|
+
#
|
|
169
|
+
# @param duration_ms [Float]
|
|
170
|
+
#
|
|
171
|
+
# @param error [String]
|
|
172
|
+
#
|
|
173
|
+
# @param method_ [String]
|
|
174
|
+
#
|
|
175
|
+
# @param target_description [String]
|
|
176
|
+
|
|
177
|
+
# Applied means the requested page state was visibly verified. Failed means it was
|
|
178
|
+
# not verified. Skipped means it was not attempted.
|
|
179
|
+
#
|
|
180
|
+
# @see ContextDev::Models::WebExtractResponse::Metadata::ActionsApplied#status
|
|
181
|
+
module Status
|
|
182
|
+
extend ContextDev::Internal::Type::Enum
|
|
183
|
+
|
|
184
|
+
APPLIED = :applied
|
|
185
|
+
FAILED = :failed
|
|
186
|
+
SKIPPED = :skipped
|
|
187
|
+
|
|
188
|
+
# @!method self.values
|
|
189
|
+
# @return [Array<Symbol>]
|
|
190
|
+
end
|
|
191
|
+
end
|
|
108
192
|
end
|
|
109
193
|
|
|
110
194
|
# @see ContextDev::Models::WebExtractResponse#key_metadata
|
|
@@ -7,6 +7,16 @@ module ContextDev
|
|
|
7
7
|
extend ContextDev::Internal::Type::RequestParameters::Converter
|
|
8
8
|
include ContextDev::Internal::Type::RequestParameters
|
|
9
9
|
|
|
10
|
+
# @!attribute clear_popups
|
|
11
|
+
# Optional parameter for comprehensive popup cleanup. If 'true', the browser
|
|
12
|
+
# dismisses detected cookie/consent UI and clears other detected obstructive
|
|
13
|
+
# popups and overlays before capture. If 'false' or not provided, this parameter
|
|
14
|
+
# requests no cleanup; handleCookiePopup can still request cookie/consent handling
|
|
15
|
+
# independently.
|
|
16
|
+
#
|
|
17
|
+
# @return [Boolean, nil]
|
|
18
|
+
optional :clear_popups, ContextDev::Internal::Type::Boolean
|
|
19
|
+
|
|
10
20
|
# @!attribute color_scheme
|
|
11
21
|
# Optional parameter to choose the site's visual theme in the screenshot. Use
|
|
12
22
|
# 'light' or 'dark' when the site offers both appearances.
|
|
@@ -121,10 +131,12 @@ module ContextDev
|
|
|
121
131
|
# @return [Symbol, ContextDev::Models::WebScreenshotParams::Zdr, nil]
|
|
122
132
|
optional :zdr, enum: -> { ContextDev::WebScreenshotParams::Zdr }
|
|
123
133
|
|
|
124
|
-
# @!method initialize(color_scheme: nil, country: nil, direct_url: nil, domain: nil, full_screenshot: nil, handle_cookie_popup: nil, max_age_ms: nil, page: nil, scroll_offset: nil, tags: nil, timeout_ms: nil, viewport: nil, wait_for_ms: nil, zdr: nil, request_options: {})
|
|
134
|
+
# @!method initialize(clear_popups: nil, color_scheme: nil, country: nil, direct_url: nil, domain: nil, full_screenshot: nil, handle_cookie_popup: nil, max_age_ms: nil, page: nil, scroll_offset: nil, tags: nil, timeout_ms: nil, viewport: nil, wait_for_ms: nil, zdr: nil, request_options: {})
|
|
125
135
|
# Some parameter documentations has been truncated, see
|
|
126
136
|
# {ContextDev::Models::WebScreenshotParams} for more details.
|
|
127
137
|
#
|
|
138
|
+
# @param clear_popups [Boolean] Optional parameter for comprehensive popup cleanup. If 'true', the browser dismi
|
|
139
|
+
#
|
|
128
140
|
# @param color_scheme [Symbol, ContextDev::Models::WebScreenshotParams::ColorScheme] Optional parameter to choose the site's visual theme in the screenshot. Use 'lig
|
|
129
141
|
#
|
|
130
142
|
# @param country [Symbol, ContextDev::Models::WebScreenshotParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al
|
|
@@ -129,7 +129,9 @@ module ContextDev
|
|
|
129
129
|
optional :timeout_ms, Integer, api_name: :timeoutMS
|
|
130
130
|
|
|
131
131
|
# @!attribute url_regex
|
|
132
|
-
# Regex pattern. Only URLs matching this pattern will be followed and scraped.
|
|
132
|
+
# Regex pattern. Only URLs matching this pattern will be followed and scraped. An
|
|
133
|
+
# automatic prefix scope in the form ^<starting URL> follows a redirect of the
|
|
134
|
+
# starting page.
|
|
133
135
|
#
|
|
134
136
|
# @return [String, nil]
|
|
135
137
|
optional :url_regex, String, api_name: :urlRegex
|
|
@@ -142,8 +144,8 @@ module ContextDev
|
|
|
142
144
|
optional :use_main_content_only, ContextDev::Internal::Type::Boolean, api_name: :useMainContentOnly
|
|
143
145
|
|
|
144
146
|
# @!attribute wait_for_ms
|
|
145
|
-
#
|
|
146
|
-
#
|
|
147
|
+
# Browser wait time in milliseconds after initial page load for each crawled page.
|
|
148
|
+
# Defaults to 3500 (3.5 seconds). Min: 0. Max: 30000 (30 seconds).
|
|
147
149
|
#
|
|
148
150
|
# @return [Integer, nil]
|
|
149
151
|
optional :wait_for_ms, Integer, api_name: :waitForMs
|
|
@@ -195,11 +197,11 @@ module ContextDev
|
|
|
195
197
|
#
|
|
196
198
|
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
197
199
|
#
|
|
198
|
-
# @param url_regex [String] Regex pattern. Only URLs matching this pattern will be followed and scraped.
|
|
200
|
+
# @param url_regex [String] Regex pattern. Only URLs matching this pattern will be followed and scraped. An
|
|
199
201
|
#
|
|
200
202
|
# @param use_main_content_only [Boolean] Extract only the main content, stripping headers, footers, sidebars, and navigat
|
|
201
203
|
#
|
|
202
|
-
# @param wait_for_ms [Integer]
|
|
204
|
+
# @param wait_for_ms [Integer] Browser wait time in milliseconds after initial page load for each crawled page.
|
|
203
205
|
#
|
|
204
206
|
# @param zdr [Symbol, ContextDev::Models::WebWebCrawlMdParams::Zdr] Set to enabled to bypass shared caches and omit request and response content fro
|
|
205
207
|
#
|
|
@@ -18,7 +18,7 @@ module ContextDev
|
|
|
18
18
|
# content is captured. Requires a paid plan. Send a JSON array in the query
|
|
19
19
|
# parameter. Maximum: 5 actions.
|
|
20
20
|
#
|
|
21
|
-
# @return [Array<ContextDev::Models::WebWebScrapeHTMLParams::Action::Wait, ContextDev::Models::WebWebScrapeHTMLParams::Action::Perform>, nil]
|
|
21
|
+
# @return [Array<ContextDev::Models::WebWebScrapeHTMLParams::Action::Wait, ContextDev::Models::WebWebScrapeHTMLParams::Action::Perform, ContextDev::Models::WebWebScrapeHTMLParams::Action::Scroll>, nil]
|
|
22
22
|
optional :actions,
|
|
23
23
|
-> { ContextDev::Internal::Type::ArrayOf[union: ContextDev::WebWebScrapeHTMLParams::Action] },
|
|
24
24
|
nil?: true
|
|
@@ -128,7 +128,7 @@ module ContextDev
|
|
|
128
128
|
#
|
|
129
129
|
# @param url [String] Full URL to scrape (must include http:// or https:// protocol)
|
|
130
130
|
#
|
|
131
|
-
# @param actions [Array<ContextDev::Models::WebWebScrapeHTMLParams::Action::Wait, ContextDev::Models::WebWebScrapeHTMLParams::Action::Perform>, nil] Optional browser actions executed in array order after the page loads and before
|
|
131
|
+
# @param actions [Array<ContextDev::Models::WebWebScrapeHTMLParams::Action::Wait, ContextDev::Models::WebWebScrapeHTMLParams::Action::Perform, ContextDev::Models::WebWebScrapeHTMLParams::Action::Scroll>, nil] Optional browser actions executed in array order after the page loads and before
|
|
132
132
|
#
|
|
133
133
|
# @param country [Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al
|
|
134
134
|
#
|
|
@@ -171,6 +171,9 @@ module ContextDev
|
|
|
171
171
|
# Resolve and perform one natural-language browser action.
|
|
172
172
|
variant :perform, -> { ContextDev::WebWebScrapeHTMLParams::Action::Perform }
|
|
173
173
|
|
|
174
|
+
# Scroll the page or a selected scrollable container, waiting adaptively for content and dimensions to settle after each iteration.
|
|
175
|
+
variant :scroll, -> { ContextDev::WebWebScrapeHTMLParams::Action::Scroll }
|
|
176
|
+
|
|
174
177
|
class Wait < ContextDev::Internal::Type::BaseModel
|
|
175
178
|
# @!attribute do_
|
|
176
179
|
#
|
|
@@ -207,8 +210,101 @@ module ContextDev
|
|
|
207
210
|
# @param do_ [Symbol, :perform]
|
|
208
211
|
end
|
|
209
212
|
|
|
213
|
+
class Scroll < ContextDev::Internal::Type::BaseModel
|
|
214
|
+
# @!attribute do_
|
|
215
|
+
#
|
|
216
|
+
# @return [Symbol, :scroll]
|
|
217
|
+
required :do_, const: :scroll, api_name: :do
|
|
218
|
+
|
|
219
|
+
# @!attribute amount
|
|
220
|
+
# Pixels per scroll, one visible viewport, or the current scroll boundary.
|
|
221
|
+
# Defaults to viewport.
|
|
222
|
+
#
|
|
223
|
+
# @return [Integer, Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Action::Scroll::Amount, nil]
|
|
224
|
+
optional :amount, union: -> { ContextDev::WebWebScrapeHTMLParams::Action::Scroll::Amount }
|
|
225
|
+
|
|
226
|
+
# @!attribute container
|
|
227
|
+
# CSS selector for the first matching scroll container. Defaults to the page.
|
|
228
|
+
#
|
|
229
|
+
# @return [String, nil]
|
|
230
|
+
optional :container, String
|
|
231
|
+
|
|
232
|
+
# @!attribute direction
|
|
233
|
+
# Direction to scroll. Defaults to down.
|
|
234
|
+
#
|
|
235
|
+
# @return [Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Action::Scroll::Direction, nil]
|
|
236
|
+
optional :direction, enum: -> { ContextDev::WebWebScrapeHTMLParams::Action::Scroll::Direction }
|
|
237
|
+
|
|
238
|
+
# @!attribute max_scrolls
|
|
239
|
+
# Maximum scroll iterations. Stops early when scrolling and scrollable extent stop
|
|
240
|
+
# changing. Defaults to 1.
|
|
241
|
+
#
|
|
242
|
+
# @return [Integer, nil]
|
|
243
|
+
optional :max_scrolls, Integer, api_name: :maxScrolls
|
|
244
|
+
|
|
245
|
+
# @!method initialize(amount: nil, container: nil, direction: nil, max_scrolls: nil, do_: :scroll)
|
|
246
|
+
# Some parameter documentations has been truncated, see
|
|
247
|
+
# {ContextDev::Models::WebWebScrapeHTMLParams::Action::Scroll} for more details.
|
|
248
|
+
#
|
|
249
|
+
# Scroll the page or a selected scrollable container, waiting adaptively for
|
|
250
|
+
# content and dimensions to settle after each iteration.
|
|
251
|
+
#
|
|
252
|
+
# @param amount [Integer, Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Action::Scroll::Amount] Pixels per scroll, one visible viewport, or the current scroll boundary. Default
|
|
253
|
+
#
|
|
254
|
+
# @param container [String] CSS selector for the first matching scroll container. Defaults to the page.
|
|
255
|
+
#
|
|
256
|
+
# @param direction [Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Action::Scroll::Direction] Direction to scroll. Defaults to down.
|
|
257
|
+
#
|
|
258
|
+
# @param max_scrolls [Integer] Maximum scroll iterations. Stops early when scrolling and scrollable extent stop
|
|
259
|
+
#
|
|
260
|
+
# @param do_ [Symbol, :scroll]
|
|
261
|
+
|
|
262
|
+
# Pixels per scroll, one visible viewport, or the current scroll boundary.
|
|
263
|
+
# Defaults to viewport.
|
|
264
|
+
#
|
|
265
|
+
# @see ContextDev::Models::WebWebScrapeHTMLParams::Action::Scroll#amount
|
|
266
|
+
module Amount
|
|
267
|
+
extend ContextDev::Internal::Type::Union
|
|
268
|
+
|
|
269
|
+
variant Integer
|
|
270
|
+
|
|
271
|
+
variant const: -> { ContextDev::Models::WebWebScrapeHTMLParams::Action::Scroll::Amount::VIEWPORT }
|
|
272
|
+
|
|
273
|
+
variant const: -> { ContextDev::Models::WebWebScrapeHTMLParams::Action::Scroll::Amount::MAX }
|
|
274
|
+
|
|
275
|
+
# @!method self.variants
|
|
276
|
+
# @return [Array(Integer, Symbol)]
|
|
277
|
+
|
|
278
|
+
define_sorbet_constant!(:Variants) do
|
|
279
|
+
T.type_alias { T.any(Integer, ContextDev::WebWebScrapeHTMLParams::Action::Scroll::Amount::TaggedSymbol) }
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
# @!group
|
|
283
|
+
|
|
284
|
+
VIEWPORT = :viewport
|
|
285
|
+
MAX = :max
|
|
286
|
+
|
|
287
|
+
# @!endgroup
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
# Direction to scroll. Defaults to down.
|
|
291
|
+
#
|
|
292
|
+
# @see ContextDev::Models::WebWebScrapeHTMLParams::Action::Scroll#direction
|
|
293
|
+
module Direction
|
|
294
|
+
extend ContextDev::Internal::Type::Enum
|
|
295
|
+
|
|
296
|
+
UP = :up
|
|
297
|
+
DOWN = :down
|
|
298
|
+
LEFT = :left
|
|
299
|
+
RIGHT = :right
|
|
300
|
+
|
|
301
|
+
# @!method self.values
|
|
302
|
+
# @return [Array<Symbol>]
|
|
303
|
+
end
|
|
304
|
+
end
|
|
305
|
+
|
|
210
306
|
# @!method self.variants
|
|
211
|
-
# @return [Array(ContextDev::Models::WebWebScrapeHTMLParams::Action::Wait, ContextDev::Models::WebWebScrapeHTMLParams::Action::Perform)]
|
|
307
|
+
# @return [Array(ContextDev::Models::WebWebScrapeHTMLParams::Action::Wait, ContextDev::Models::WebWebScrapeHTMLParams::Action::Perform, ContextDev::Models::WebWebScrapeHTMLParams::Action::Scroll)]
|
|
212
308
|
end
|
|
213
309
|
|
|
214
310
|
# Fetch the target page through a residential proxy in this country (ISO 3166-1
|
|
@@ -18,7 +18,7 @@ module ContextDev
|
|
|
18
18
|
# content is captured. Requires a paid plan. Send a JSON array in the query
|
|
19
19
|
# parameter. Maximum: 5 actions.
|
|
20
20
|
#
|
|
21
|
-
# @return [Array<ContextDev::Models::WebWebScrapeImagesParams::Action::Wait, ContextDev::Models::WebWebScrapeImagesParams::Action::Perform>, nil]
|
|
21
|
+
# @return [Array<ContextDev::Models::WebWebScrapeImagesParams::Action::Wait, ContextDev::Models::WebWebScrapeImagesParams::Action::Perform, ContextDev::Models::WebWebScrapeImagesParams::Action::Scroll>, nil]
|
|
22
22
|
optional :actions,
|
|
23
23
|
-> {
|
|
24
24
|
ContextDev::Internal::Type::ArrayOf[union: ContextDev::WebWebScrapeImagesParams::Action]
|
|
@@ -85,7 +85,7 @@ module ContextDev
|
|
|
85
85
|
#
|
|
86
86
|
# @param url [String] Page URL to inspect. Must include http:// or https://.
|
|
87
87
|
#
|
|
88
|
-
# @param actions [Array<ContextDev::Models::WebWebScrapeImagesParams::Action::Wait, ContextDev::Models::WebWebScrapeImagesParams::Action::Perform>, nil] Optional browser actions executed in array order after the page loads and before
|
|
88
|
+
# @param actions [Array<ContextDev::Models::WebWebScrapeImagesParams::Action::Wait, ContextDev::Models::WebWebScrapeImagesParams::Action::Perform, ContextDev::Models::WebWebScrapeImagesParams::Action::Scroll>, nil] Optional browser actions executed in array order after the page loads and before
|
|
89
89
|
#
|
|
90
90
|
# @param dedupe [Boolean] When true, visually duplicate images are removed: every image is loaded and perc
|
|
91
91
|
#
|
|
@@ -116,6 +116,9 @@ module ContextDev
|
|
|
116
116
|
# Resolve and perform one natural-language browser action.
|
|
117
117
|
variant :perform, -> { ContextDev::WebWebScrapeImagesParams::Action::Perform }
|
|
118
118
|
|
|
119
|
+
# Scroll the page or a selected scrollable container, waiting adaptively for content and dimensions to settle after each iteration.
|
|
120
|
+
variant :scroll, -> { ContextDev::WebWebScrapeImagesParams::Action::Scroll }
|
|
121
|
+
|
|
119
122
|
class Wait < ContextDev::Internal::Type::BaseModel
|
|
120
123
|
# @!attribute do_
|
|
121
124
|
#
|
|
@@ -152,8 +155,101 @@ module ContextDev
|
|
|
152
155
|
# @param do_ [Symbol, :perform]
|
|
153
156
|
end
|
|
154
157
|
|
|
158
|
+
class Scroll < ContextDev::Internal::Type::BaseModel
|
|
159
|
+
# @!attribute do_
|
|
160
|
+
#
|
|
161
|
+
# @return [Symbol, :scroll]
|
|
162
|
+
required :do_, const: :scroll, api_name: :do
|
|
163
|
+
|
|
164
|
+
# @!attribute amount
|
|
165
|
+
# Pixels per scroll, one visible viewport, or the current scroll boundary.
|
|
166
|
+
# Defaults to viewport.
|
|
167
|
+
#
|
|
168
|
+
# @return [Integer, Symbol, ContextDev::Models::WebWebScrapeImagesParams::Action::Scroll::Amount, nil]
|
|
169
|
+
optional :amount, union: -> { ContextDev::WebWebScrapeImagesParams::Action::Scroll::Amount }
|
|
170
|
+
|
|
171
|
+
# @!attribute container
|
|
172
|
+
# CSS selector for the first matching scroll container. Defaults to the page.
|
|
173
|
+
#
|
|
174
|
+
# @return [String, nil]
|
|
175
|
+
optional :container, String
|
|
176
|
+
|
|
177
|
+
# @!attribute direction
|
|
178
|
+
# Direction to scroll. Defaults to down.
|
|
179
|
+
#
|
|
180
|
+
# @return [Symbol, ContextDev::Models::WebWebScrapeImagesParams::Action::Scroll::Direction, nil]
|
|
181
|
+
optional :direction, enum: -> { ContextDev::WebWebScrapeImagesParams::Action::Scroll::Direction }
|
|
182
|
+
|
|
183
|
+
# @!attribute max_scrolls
|
|
184
|
+
# Maximum scroll iterations. Stops early when scrolling and scrollable extent stop
|
|
185
|
+
# changing. Defaults to 1.
|
|
186
|
+
#
|
|
187
|
+
# @return [Integer, nil]
|
|
188
|
+
optional :max_scrolls, Integer, api_name: :maxScrolls
|
|
189
|
+
|
|
190
|
+
# @!method initialize(amount: nil, container: nil, direction: nil, max_scrolls: nil, do_: :scroll)
|
|
191
|
+
# Some parameter documentations has been truncated, see
|
|
192
|
+
# {ContextDev::Models::WebWebScrapeImagesParams::Action::Scroll} for more details.
|
|
193
|
+
#
|
|
194
|
+
# Scroll the page or a selected scrollable container, waiting adaptively for
|
|
195
|
+
# content and dimensions to settle after each iteration.
|
|
196
|
+
#
|
|
197
|
+
# @param amount [Integer, Symbol, ContextDev::Models::WebWebScrapeImagesParams::Action::Scroll::Amount] Pixels per scroll, one visible viewport, or the current scroll boundary. Default
|
|
198
|
+
#
|
|
199
|
+
# @param container [String] CSS selector for the first matching scroll container. Defaults to the page.
|
|
200
|
+
#
|
|
201
|
+
# @param direction [Symbol, ContextDev::Models::WebWebScrapeImagesParams::Action::Scroll::Direction] Direction to scroll. Defaults to down.
|
|
202
|
+
#
|
|
203
|
+
# @param max_scrolls [Integer] Maximum scroll iterations. Stops early when scrolling and scrollable extent stop
|
|
204
|
+
#
|
|
205
|
+
# @param do_ [Symbol, :scroll]
|
|
206
|
+
|
|
207
|
+
# Pixels per scroll, one visible viewport, or the current scroll boundary.
|
|
208
|
+
# Defaults to viewport.
|
|
209
|
+
#
|
|
210
|
+
# @see ContextDev::Models::WebWebScrapeImagesParams::Action::Scroll#amount
|
|
211
|
+
module Amount
|
|
212
|
+
extend ContextDev::Internal::Type::Union
|
|
213
|
+
|
|
214
|
+
variant Integer
|
|
215
|
+
|
|
216
|
+
variant const: -> { ContextDev::Models::WebWebScrapeImagesParams::Action::Scroll::Amount::VIEWPORT }
|
|
217
|
+
|
|
218
|
+
variant const: -> { ContextDev::Models::WebWebScrapeImagesParams::Action::Scroll::Amount::MAX }
|
|
219
|
+
|
|
220
|
+
# @!method self.variants
|
|
221
|
+
# @return [Array(Integer, Symbol)]
|
|
222
|
+
|
|
223
|
+
define_sorbet_constant!(:Variants) do
|
|
224
|
+
T.type_alias { T.any(Integer, ContextDev::WebWebScrapeImagesParams::Action::Scroll::Amount::TaggedSymbol) }
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
# @!group
|
|
228
|
+
|
|
229
|
+
VIEWPORT = :viewport
|
|
230
|
+
MAX = :max
|
|
231
|
+
|
|
232
|
+
# @!endgroup
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
# Direction to scroll. Defaults to down.
|
|
236
|
+
#
|
|
237
|
+
# @see ContextDev::Models::WebWebScrapeImagesParams::Action::Scroll#direction
|
|
238
|
+
module Direction
|
|
239
|
+
extend ContextDev::Internal::Type::Enum
|
|
240
|
+
|
|
241
|
+
UP = :up
|
|
242
|
+
DOWN = :down
|
|
243
|
+
LEFT = :left
|
|
244
|
+
RIGHT = :right
|
|
245
|
+
|
|
246
|
+
# @!method self.values
|
|
247
|
+
# @return [Array<Symbol>]
|
|
248
|
+
end
|
|
249
|
+
end
|
|
250
|
+
|
|
155
251
|
# @!method self.variants
|
|
156
|
-
# @return [Array(ContextDev::Models::WebWebScrapeImagesParams::Action::Wait, ContextDev::Models::WebWebScrapeImagesParams::Action::Perform)]
|
|
252
|
+
# @return [Array(ContextDev::Models::WebWebScrapeImagesParams::Action::Wait, ContextDev::Models::WebWebScrapeImagesParams::Action::Perform, ContextDev::Models::WebWebScrapeImagesParams::Action::Scroll)]
|
|
157
253
|
end
|
|
158
254
|
|
|
159
255
|
class Enrichment < ContextDev::Internal::Type::BaseModel
|