context.dev 2.11.0 → 2.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +11 -0
- data/README.md +1 -1
- data/lib/context_dev/models/ai_extract_product_response.rb +48 -1
- data/lib/context_dev/models/ai_extract_products_response.rb +48 -1
- data/lib/context_dev/models/news_search_params.rb +1 -0
- data/lib/context_dev/models/web_extract_params.rb +169 -6
- data/lib/context_dev/models/web_extract_response.rb +85 -1
- data/lib/context_dev/models/web_web_crawl_md_params.rb +7 -5
- data/lib/context_dev/models/web_web_scrape_html_params.rb +99 -3
- data/lib/context_dev/models/web_web_scrape_images_params.rb +99 -3
- data/lib/context_dev/models/web_web_scrape_images_response.rb +85 -1
- data/lib/context_dev/models/web_web_scrape_md_params.rb +99 -3
- data/lib/context_dev/resources/web.rb +9 -7
- data/lib/context_dev/version.rb +1 -1
- data/rbi/context_dev/models/ai_extract_product_response.rbi +99 -0
- data/rbi/context_dev/models/ai_extract_products_response.rbi +99 -0
- data/rbi/context_dev/models/news_search_params.rbi +5 -0
- data/rbi/context_dev/models/web_extract_params.rbi +353 -8
- data/rbi/context_dev/models/web_extract_response.rbi +172 -3
- data/rbi/context_dev/models/web_web_crawl_md_params.rbi +10 -6
- data/rbi/context_dev/models/web_web_scrape_html_params.rbi +218 -4
- data/rbi/context_dev/models/web_web_scrape_images_params.rbi +218 -4
- data/rbi/context_dev/models/web_web_scrape_images_response.rbi +169 -0
- data/rbi/context_dev/models/web_web_scrape_md_params.rbi +218 -4
- data/rbi/context_dev/resources/web.rbi +31 -10
- data/sig/context_dev/models/ai_extract_product_response.rbs +40 -0
- data/sig/context_dev/models/ai_extract_products_response.rbs +40 -0
- data/sig/context_dev/models/news_search_params.rbs +2 -0
- data/sig/context_dev/models/web_extract_params.rbs +117 -0
- data/sig/context_dev/models/web_extract_response.rbs +81 -3
- data/sig/context_dev/models/web_web_scrape_html_params.rbs +74 -0
- data/sig/context_dev/models/web_web_scrape_images_params.rbs +74 -0
- data/sig/context_dev/models/web_web_scrape_images_response.rbs +78 -0
- data/sig/context_dev/models/web_web_scrape_md_params.rbs +74 -0
- data/sig/context_dev/resources/web.rbs +1 -0
- metadata +2 -2
|
@@ -18,7 +18,7 @@ module ContextDev
|
|
|
18
18
|
# content is captured. Requires a paid plan. Send a JSON array in the query
|
|
19
19
|
# parameter. Maximum: 5 actions.
|
|
20
20
|
#
|
|
21
|
-
# @return [Array<ContextDev::Models::WebWebScrapeHTMLParams::Action::Wait, ContextDev::Models::WebWebScrapeHTMLParams::Action::Perform>, nil]
|
|
21
|
+
# @return [Array<ContextDev::Models::WebWebScrapeHTMLParams::Action::Wait, ContextDev::Models::WebWebScrapeHTMLParams::Action::Perform, ContextDev::Models::WebWebScrapeHTMLParams::Action::Scroll>, nil]
|
|
22
22
|
optional :actions,
|
|
23
23
|
-> { ContextDev::Internal::Type::ArrayOf[union: ContextDev::WebWebScrapeHTMLParams::Action] },
|
|
24
24
|
nil?: true
|
|
@@ -128,7 +128,7 @@ module ContextDev
|
|
|
128
128
|
#
|
|
129
129
|
# @param url [String] Full URL to scrape (must include http:// or https:// protocol)
|
|
130
130
|
#
|
|
131
|
-
# @param actions [Array<ContextDev::Models::WebWebScrapeHTMLParams::Action::Wait, ContextDev::Models::WebWebScrapeHTMLParams::Action::Perform>, nil] Optional browser actions executed in array order after the page loads and before
|
|
131
|
+
# @param actions [Array<ContextDev::Models::WebWebScrapeHTMLParams::Action::Wait, ContextDev::Models::WebWebScrapeHTMLParams::Action::Perform, ContextDev::Models::WebWebScrapeHTMLParams::Action::Scroll>, nil] Optional browser actions executed in array order after the page loads and before
|
|
132
132
|
#
|
|
133
133
|
# @param country [Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al
|
|
134
134
|
#
|
|
@@ -171,6 +171,9 @@ module ContextDev
|
|
|
171
171
|
# Resolve and perform one natural-language browser action.
|
|
172
172
|
variant :perform, -> { ContextDev::WebWebScrapeHTMLParams::Action::Perform }
|
|
173
173
|
|
|
174
|
+
# Scroll the page or a selected scrollable container, waiting adaptively for content and dimensions to settle after each iteration.
|
|
175
|
+
variant :scroll, -> { ContextDev::WebWebScrapeHTMLParams::Action::Scroll }
|
|
176
|
+
|
|
174
177
|
class Wait < ContextDev::Internal::Type::BaseModel
|
|
175
178
|
# @!attribute do_
|
|
176
179
|
#
|
|
@@ -207,8 +210,101 @@ module ContextDev
|
|
|
207
210
|
# @param do_ [Symbol, :perform]
|
|
208
211
|
end
|
|
209
212
|
|
|
213
|
+
class Scroll < ContextDev::Internal::Type::BaseModel
|
|
214
|
+
# @!attribute do_
|
|
215
|
+
#
|
|
216
|
+
# @return [Symbol, :scroll]
|
|
217
|
+
required :do_, const: :scroll, api_name: :do
|
|
218
|
+
|
|
219
|
+
# @!attribute amount
|
|
220
|
+
# Pixels per scroll, one visible viewport, or the current scroll boundary.
|
|
221
|
+
# Defaults to viewport.
|
|
222
|
+
#
|
|
223
|
+
# @return [Integer, Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Action::Scroll::Amount, nil]
|
|
224
|
+
optional :amount, union: -> { ContextDev::WebWebScrapeHTMLParams::Action::Scroll::Amount }
|
|
225
|
+
|
|
226
|
+
# @!attribute container
|
|
227
|
+
# CSS selector for the first matching scroll container. Defaults to the page.
|
|
228
|
+
#
|
|
229
|
+
# @return [String, nil]
|
|
230
|
+
optional :container, String
|
|
231
|
+
|
|
232
|
+
# @!attribute direction
|
|
233
|
+
# Direction to scroll. Defaults to down.
|
|
234
|
+
#
|
|
235
|
+
# @return [Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Action::Scroll::Direction, nil]
|
|
236
|
+
optional :direction, enum: -> { ContextDev::WebWebScrapeHTMLParams::Action::Scroll::Direction }
|
|
237
|
+
|
|
238
|
+
# @!attribute max_scrolls
|
|
239
|
+
# Maximum scroll iterations. Stops early when scrolling and scrollable extent stop
|
|
240
|
+
# changing. Defaults to 1.
|
|
241
|
+
#
|
|
242
|
+
# @return [Integer, nil]
|
|
243
|
+
optional :max_scrolls, Integer, api_name: :maxScrolls
|
|
244
|
+
|
|
245
|
+
# @!method initialize(amount: nil, container: nil, direction: nil, max_scrolls: nil, do_: :scroll)
|
|
246
|
+
# Some parameter documentations has been truncated, see
|
|
247
|
+
# {ContextDev::Models::WebWebScrapeHTMLParams::Action::Scroll} for more details.
|
|
248
|
+
#
|
|
249
|
+
# Scroll the page or a selected scrollable container, waiting adaptively for
|
|
250
|
+
# content and dimensions to settle after each iteration.
|
|
251
|
+
#
|
|
252
|
+
# @param amount [Integer, Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Action::Scroll::Amount] Pixels per scroll, one visible viewport, or the current scroll boundary. Default
|
|
253
|
+
#
|
|
254
|
+
# @param container [String] CSS selector for the first matching scroll container. Defaults to the page.
|
|
255
|
+
#
|
|
256
|
+
# @param direction [Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Action::Scroll::Direction] Direction to scroll. Defaults to down.
|
|
257
|
+
#
|
|
258
|
+
# @param max_scrolls [Integer] Maximum scroll iterations. Stops early when scrolling and scrollable extent stop
|
|
259
|
+
#
|
|
260
|
+
# @param do_ [Symbol, :scroll]
|
|
261
|
+
|
|
262
|
+
# Pixels per scroll, one visible viewport, or the current scroll boundary.
|
|
263
|
+
# Defaults to viewport.
|
|
264
|
+
#
|
|
265
|
+
# @see ContextDev::Models::WebWebScrapeHTMLParams::Action::Scroll#amount
|
|
266
|
+
module Amount
|
|
267
|
+
extend ContextDev::Internal::Type::Union
|
|
268
|
+
|
|
269
|
+
variant Integer
|
|
270
|
+
|
|
271
|
+
variant const: -> { ContextDev::Models::WebWebScrapeHTMLParams::Action::Scroll::Amount::VIEWPORT }
|
|
272
|
+
|
|
273
|
+
variant const: -> { ContextDev::Models::WebWebScrapeHTMLParams::Action::Scroll::Amount::MAX }
|
|
274
|
+
|
|
275
|
+
# @!method self.variants
|
|
276
|
+
# @return [Array(Integer, Symbol)]
|
|
277
|
+
|
|
278
|
+
define_sorbet_constant!(:Variants) do
|
|
279
|
+
T.type_alias { T.any(Integer, ContextDev::WebWebScrapeHTMLParams::Action::Scroll::Amount::TaggedSymbol) }
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
# @!group
|
|
283
|
+
|
|
284
|
+
VIEWPORT = :viewport
|
|
285
|
+
MAX = :max
|
|
286
|
+
|
|
287
|
+
# @!endgroup
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
# Direction to scroll. Defaults to down.
|
|
291
|
+
#
|
|
292
|
+
# @see ContextDev::Models::WebWebScrapeHTMLParams::Action::Scroll#direction
|
|
293
|
+
module Direction
|
|
294
|
+
extend ContextDev::Internal::Type::Enum
|
|
295
|
+
|
|
296
|
+
UP = :up
|
|
297
|
+
DOWN = :down
|
|
298
|
+
LEFT = :left
|
|
299
|
+
RIGHT = :right
|
|
300
|
+
|
|
301
|
+
# @!method self.values
|
|
302
|
+
# @return [Array<Symbol>]
|
|
303
|
+
end
|
|
304
|
+
end
|
|
305
|
+
|
|
210
306
|
# @!method self.variants
|
|
211
|
-
# @return [Array(ContextDev::Models::WebWebScrapeHTMLParams::Action::Wait, ContextDev::Models::WebWebScrapeHTMLParams::Action::Perform)]
|
|
307
|
+
# @return [Array(ContextDev::Models::WebWebScrapeHTMLParams::Action::Wait, ContextDev::Models::WebWebScrapeHTMLParams::Action::Perform, ContextDev::Models::WebWebScrapeHTMLParams::Action::Scroll)]
|
|
212
308
|
end
|
|
213
309
|
|
|
214
310
|
# Fetch the target page through a residential proxy in this country (ISO 3166-1
|
|
@@ -18,7 +18,7 @@ module ContextDev
|
|
|
18
18
|
# content is captured. Requires a paid plan. Send a JSON array in the query
|
|
19
19
|
# parameter. Maximum: 5 actions.
|
|
20
20
|
#
|
|
21
|
-
# @return [Array<ContextDev::Models::WebWebScrapeImagesParams::Action::Wait, ContextDev::Models::WebWebScrapeImagesParams::Action::Perform>, nil]
|
|
21
|
+
# @return [Array<ContextDev::Models::WebWebScrapeImagesParams::Action::Wait, ContextDev::Models::WebWebScrapeImagesParams::Action::Perform, ContextDev::Models::WebWebScrapeImagesParams::Action::Scroll>, nil]
|
|
22
22
|
optional :actions,
|
|
23
23
|
-> {
|
|
24
24
|
ContextDev::Internal::Type::ArrayOf[union: ContextDev::WebWebScrapeImagesParams::Action]
|
|
@@ -85,7 +85,7 @@ module ContextDev
|
|
|
85
85
|
#
|
|
86
86
|
# @param url [String] Page URL to inspect. Must include http:// or https://.
|
|
87
87
|
#
|
|
88
|
-
# @param actions [Array<ContextDev::Models::WebWebScrapeImagesParams::Action::Wait, ContextDev::Models::WebWebScrapeImagesParams::Action::Perform>, nil] Optional browser actions executed in array order after the page loads and before
|
|
88
|
+
# @param actions [Array<ContextDev::Models::WebWebScrapeImagesParams::Action::Wait, ContextDev::Models::WebWebScrapeImagesParams::Action::Perform, ContextDev::Models::WebWebScrapeImagesParams::Action::Scroll>, nil] Optional browser actions executed in array order after the page loads and before
|
|
89
89
|
#
|
|
90
90
|
# @param dedupe [Boolean] When true, visually duplicate images are removed: every image is loaded and perc
|
|
91
91
|
#
|
|
@@ -116,6 +116,9 @@ module ContextDev
|
|
|
116
116
|
# Resolve and perform one natural-language browser action.
|
|
117
117
|
variant :perform, -> { ContextDev::WebWebScrapeImagesParams::Action::Perform }
|
|
118
118
|
|
|
119
|
+
# Scroll the page or a selected scrollable container, waiting adaptively for content and dimensions to settle after each iteration.
|
|
120
|
+
variant :scroll, -> { ContextDev::WebWebScrapeImagesParams::Action::Scroll }
|
|
121
|
+
|
|
119
122
|
class Wait < ContextDev::Internal::Type::BaseModel
|
|
120
123
|
# @!attribute do_
|
|
121
124
|
#
|
|
@@ -152,8 +155,101 @@ module ContextDev
|
|
|
152
155
|
# @param do_ [Symbol, :perform]
|
|
153
156
|
end
|
|
154
157
|
|
|
158
|
+
class Scroll < ContextDev::Internal::Type::BaseModel
|
|
159
|
+
# @!attribute do_
|
|
160
|
+
#
|
|
161
|
+
# @return [Symbol, :scroll]
|
|
162
|
+
required :do_, const: :scroll, api_name: :do
|
|
163
|
+
|
|
164
|
+
# @!attribute amount
|
|
165
|
+
# Pixels per scroll, one visible viewport, or the current scroll boundary.
|
|
166
|
+
# Defaults to viewport.
|
|
167
|
+
#
|
|
168
|
+
# @return [Integer, Symbol, ContextDev::Models::WebWebScrapeImagesParams::Action::Scroll::Amount, nil]
|
|
169
|
+
optional :amount, union: -> { ContextDev::WebWebScrapeImagesParams::Action::Scroll::Amount }
|
|
170
|
+
|
|
171
|
+
# @!attribute container
|
|
172
|
+
# CSS selector for the first matching scroll container. Defaults to the page.
|
|
173
|
+
#
|
|
174
|
+
# @return [String, nil]
|
|
175
|
+
optional :container, String
|
|
176
|
+
|
|
177
|
+
# @!attribute direction
|
|
178
|
+
# Direction to scroll. Defaults to down.
|
|
179
|
+
#
|
|
180
|
+
# @return [Symbol, ContextDev::Models::WebWebScrapeImagesParams::Action::Scroll::Direction, nil]
|
|
181
|
+
optional :direction, enum: -> { ContextDev::WebWebScrapeImagesParams::Action::Scroll::Direction }
|
|
182
|
+
|
|
183
|
+
# @!attribute max_scrolls
|
|
184
|
+
# Maximum scroll iterations. Stops early when scrolling and scrollable extent stop
|
|
185
|
+
# changing. Defaults to 1.
|
|
186
|
+
#
|
|
187
|
+
# @return [Integer, nil]
|
|
188
|
+
optional :max_scrolls, Integer, api_name: :maxScrolls
|
|
189
|
+
|
|
190
|
+
# @!method initialize(amount: nil, container: nil, direction: nil, max_scrolls: nil, do_: :scroll)
|
|
191
|
+
# Some parameter documentations has been truncated, see
|
|
192
|
+
# {ContextDev::Models::WebWebScrapeImagesParams::Action::Scroll} for more details.
|
|
193
|
+
#
|
|
194
|
+
# Scroll the page or a selected scrollable container, waiting adaptively for
|
|
195
|
+
# content and dimensions to settle after each iteration.
|
|
196
|
+
#
|
|
197
|
+
# @param amount [Integer, Symbol, ContextDev::Models::WebWebScrapeImagesParams::Action::Scroll::Amount] Pixels per scroll, one visible viewport, or the current scroll boundary. Default
|
|
198
|
+
#
|
|
199
|
+
# @param container [String] CSS selector for the first matching scroll container. Defaults to the page.
|
|
200
|
+
#
|
|
201
|
+
# @param direction [Symbol, ContextDev::Models::WebWebScrapeImagesParams::Action::Scroll::Direction] Direction to scroll. Defaults to down.
|
|
202
|
+
#
|
|
203
|
+
# @param max_scrolls [Integer] Maximum scroll iterations. Stops early when scrolling and scrollable extent stop
|
|
204
|
+
#
|
|
205
|
+
# @param do_ [Symbol, :scroll]
|
|
206
|
+
|
|
207
|
+
# Pixels per scroll, one visible viewport, or the current scroll boundary.
|
|
208
|
+
# Defaults to viewport.
|
|
209
|
+
#
|
|
210
|
+
# @see ContextDev::Models::WebWebScrapeImagesParams::Action::Scroll#amount
|
|
211
|
+
module Amount
|
|
212
|
+
extend ContextDev::Internal::Type::Union
|
|
213
|
+
|
|
214
|
+
variant Integer
|
|
215
|
+
|
|
216
|
+
variant const: -> { ContextDev::Models::WebWebScrapeImagesParams::Action::Scroll::Amount::VIEWPORT }
|
|
217
|
+
|
|
218
|
+
variant const: -> { ContextDev::Models::WebWebScrapeImagesParams::Action::Scroll::Amount::MAX }
|
|
219
|
+
|
|
220
|
+
# @!method self.variants
|
|
221
|
+
# @return [Array(Integer, Symbol)]
|
|
222
|
+
|
|
223
|
+
define_sorbet_constant!(:Variants) do
|
|
224
|
+
T.type_alias { T.any(Integer, ContextDev::WebWebScrapeImagesParams::Action::Scroll::Amount::TaggedSymbol) }
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
# @!group
|
|
228
|
+
|
|
229
|
+
VIEWPORT = :viewport
|
|
230
|
+
MAX = :max
|
|
231
|
+
|
|
232
|
+
# @!endgroup
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
# Direction to scroll. Defaults to down.
|
|
236
|
+
#
|
|
237
|
+
# @see ContextDev::Models::WebWebScrapeImagesParams::Action::Scroll#direction
|
|
238
|
+
module Direction
|
|
239
|
+
extend ContextDev::Internal::Type::Enum
|
|
240
|
+
|
|
241
|
+
UP = :up
|
|
242
|
+
DOWN = :down
|
|
243
|
+
LEFT = :left
|
|
244
|
+
RIGHT = :right
|
|
245
|
+
|
|
246
|
+
# @!method self.values
|
|
247
|
+
# @return [Array<Symbol>]
|
|
248
|
+
end
|
|
249
|
+
end
|
|
250
|
+
|
|
155
251
|
# @!method self.variants
|
|
156
|
-
# @return [Array(ContextDev::Models::WebWebScrapeImagesParams::Action::Wait, ContextDev::Models::WebWebScrapeImagesParams::Action::Perform)]
|
|
252
|
+
# @return [Array(ContextDev::Models::WebWebScrapeImagesParams::Action::Wait, ContextDev::Models::WebWebScrapeImagesParams::Action::Perform, ContextDev::Models::WebWebScrapeImagesParams::Action::Scroll)]
|
|
157
253
|
end
|
|
158
254
|
|
|
159
255
|
class Enrichment < ContextDev::Internal::Type::BaseModel
|
|
@@ -23,6 +23,14 @@ module ContextDev
|
|
|
23
23
|
# @return [String]
|
|
24
24
|
required :url, String
|
|
25
25
|
|
|
26
|
+
# @!attribute actions_applied
|
|
27
|
+
# One verified outcome per requested browser action, in request order.
|
|
28
|
+
#
|
|
29
|
+
# @return [Array<ContextDev::Models::WebWebScrapeImagesResponse::ActionsApplied>, nil]
|
|
30
|
+
optional :actions_applied,
|
|
31
|
+
-> { ContextDev::Internal::Type::ArrayOf[ContextDev::Models::WebWebScrapeImagesResponse::ActionsApplied] },
|
|
32
|
+
api_name: :actionsApplied
|
|
33
|
+
|
|
26
34
|
# @!attribute key_metadata
|
|
27
35
|
# Metadata about the API key used for the request. Included in every response
|
|
28
36
|
# whenever a valid API key is provided, even when the response status is not 200.
|
|
@@ -30,7 +38,7 @@ module ContextDev
|
|
|
30
38
|
# @return [ContextDev::Models::WebWebScrapeImagesResponse::KeyMetadata, nil]
|
|
31
39
|
optional :key_metadata, -> { ContextDev::Models::WebWebScrapeImagesResponse::KeyMetadata }
|
|
32
40
|
|
|
33
|
-
# @!method initialize(images:, success:, url:, key_metadata: nil)
|
|
41
|
+
# @!method initialize(images:, success:, url:, actions_applied: nil, key_metadata: nil)
|
|
34
42
|
# Some parameter documentations has been truncated, see
|
|
35
43
|
# {ContextDev::Models::WebWebScrapeImagesResponse} for more details.
|
|
36
44
|
#
|
|
@@ -40,6 +48,8 @@ module ContextDev
|
|
|
40
48
|
#
|
|
41
49
|
# @param url [String] Page URL that was scraped.
|
|
42
50
|
#
|
|
51
|
+
# @param actions_applied [Array<ContextDev::Models::WebWebScrapeImagesResponse::ActionsApplied>] One verified outcome per requested browser action, in request order.
|
|
52
|
+
#
|
|
43
53
|
# @param key_metadata [ContextDev::Models::WebWebScrapeImagesResponse::KeyMetadata] Metadata about the API key used for the request. Included in every response when
|
|
44
54
|
|
|
45
55
|
class Image < ContextDev::Internal::Type::BaseModel
|
|
@@ -196,6 +206,80 @@ module ContextDev
|
|
|
196
206
|
# @return [Array<Boolean>]
|
|
197
207
|
end
|
|
198
208
|
|
|
209
|
+
class ActionsApplied < ContextDev::Internal::Type::BaseModel
|
|
210
|
+
# @!attribute instruction
|
|
211
|
+
#
|
|
212
|
+
# @return [String]
|
|
213
|
+
required :instruction, String
|
|
214
|
+
|
|
215
|
+
# @!attribute status
|
|
216
|
+
# Applied means the requested page state was visibly verified. Failed means it was
|
|
217
|
+
# not verified. Skipped means it was not attempted.
|
|
218
|
+
#
|
|
219
|
+
# @return [Symbol, ContextDev::Models::WebWebScrapeImagesResponse::ActionsApplied::Status]
|
|
220
|
+
required :status, enum: -> { ContextDev::Models::WebWebScrapeImagesResponse::ActionsApplied::Status }
|
|
221
|
+
|
|
222
|
+
# @!attribute completion_evidence
|
|
223
|
+
# Visible page evidence used to verify an applied action.
|
|
224
|
+
#
|
|
225
|
+
# @return [String, nil]
|
|
226
|
+
optional :completion_evidence, String, api_name: :completionEvidence
|
|
227
|
+
|
|
228
|
+
# @!attribute duration_ms
|
|
229
|
+
#
|
|
230
|
+
# @return [Float, nil]
|
|
231
|
+
optional :duration_ms, Float, api_name: :durationMs
|
|
232
|
+
|
|
233
|
+
# @!attribute error
|
|
234
|
+
#
|
|
235
|
+
# @return [String, nil]
|
|
236
|
+
optional :error, String
|
|
237
|
+
|
|
238
|
+
# @!attribute method_
|
|
239
|
+
#
|
|
240
|
+
# @return [String, nil]
|
|
241
|
+
optional :method_, String, api_name: :method
|
|
242
|
+
|
|
243
|
+
# @!attribute target_description
|
|
244
|
+
#
|
|
245
|
+
# @return [String, nil]
|
|
246
|
+
optional :target_description, String, api_name: :targetDescription
|
|
247
|
+
|
|
248
|
+
# @!method initialize(instruction:, status:, completion_evidence: nil, duration_ms: nil, error: nil, method_: nil, target_description: nil)
|
|
249
|
+
# Some parameter documentations has been truncated, see
|
|
250
|
+
# {ContextDev::Models::WebWebScrapeImagesResponse::ActionsApplied} for more
|
|
251
|
+
# details.
|
|
252
|
+
#
|
|
253
|
+
# @param instruction [String]
|
|
254
|
+
#
|
|
255
|
+
# @param status [Symbol, ContextDev::Models::WebWebScrapeImagesResponse::ActionsApplied::Status] Applied means the requested page state was visibly verified. Failed means it was
|
|
256
|
+
#
|
|
257
|
+
# @param completion_evidence [String] Visible page evidence used to verify an applied action.
|
|
258
|
+
#
|
|
259
|
+
# @param duration_ms [Float]
|
|
260
|
+
#
|
|
261
|
+
# @param error [String]
|
|
262
|
+
#
|
|
263
|
+
# @param method_ [String]
|
|
264
|
+
#
|
|
265
|
+
# @param target_description [String]
|
|
266
|
+
|
|
267
|
+
# Applied means the requested page state was visibly verified. Failed means it was
|
|
268
|
+
# not verified. Skipped means it was not attempted.
|
|
269
|
+
#
|
|
270
|
+
# @see ContextDev::Models::WebWebScrapeImagesResponse::ActionsApplied#status
|
|
271
|
+
module Status
|
|
272
|
+
extend ContextDev::Internal::Type::Enum
|
|
273
|
+
|
|
274
|
+
APPLIED = :applied
|
|
275
|
+
FAILED = :failed
|
|
276
|
+
SKIPPED = :skipped
|
|
277
|
+
|
|
278
|
+
# @!method self.values
|
|
279
|
+
# @return [Array<Symbol>]
|
|
280
|
+
end
|
|
281
|
+
end
|
|
282
|
+
|
|
199
283
|
# @see ContextDev::Models::WebWebScrapeImagesResponse#key_metadata
|
|
200
284
|
class KeyMetadata < ContextDev::Internal::Type::BaseModel
|
|
201
285
|
# @!attribute credits_consumed
|
|
@@ -19,7 +19,7 @@ module ContextDev
|
|
|
19
19
|
# content is captured. Requires a paid plan. Send a JSON array in the query
|
|
20
20
|
# parameter. Maximum: 5 actions.
|
|
21
21
|
#
|
|
22
|
-
# @return [Array<ContextDev::Models::WebWebScrapeMdParams::Action::Wait, ContextDev::Models::WebWebScrapeMdParams::Action::Perform>, nil]
|
|
22
|
+
# @return [Array<ContextDev::Models::WebWebScrapeMdParams::Action::Wait, ContextDev::Models::WebWebScrapeMdParams::Action::Perform, ContextDev::Models::WebWebScrapeMdParams::Action::Scroll>, nil]
|
|
23
23
|
optional :actions,
|
|
24
24
|
-> { ContextDev::Internal::Type::ArrayOf[union: ContextDev::WebWebScrapeMdParams::Action] },
|
|
25
25
|
nil?: true
|
|
@@ -155,7 +155,7 @@ module ContextDev
|
|
|
155
155
|
#
|
|
156
156
|
# @param url [String] Full URL to scrape into LLM usable Markdown (must include http:// or https:// pr
|
|
157
157
|
#
|
|
158
|
-
# @param actions [Array<ContextDev::Models::WebWebScrapeMdParams::Action::Wait, ContextDev::Models::WebWebScrapeMdParams::Action::Perform>, nil] Optional browser actions executed in array order after the page loads and before
|
|
158
|
+
# @param actions [Array<ContextDev::Models::WebWebScrapeMdParams::Action::Wait, ContextDev::Models::WebWebScrapeMdParams::Action::Perform, ContextDev::Models::WebWebScrapeMdParams::Action::Scroll>, nil] Optional browser actions executed in array order after the page loads and before
|
|
159
159
|
#
|
|
160
160
|
# @param country [Symbol, ContextDev::Models::WebWebScrapeMdParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al
|
|
161
161
|
#
|
|
@@ -206,6 +206,9 @@ module ContextDev
|
|
|
206
206
|
# Resolve and perform one natural-language browser action.
|
|
207
207
|
variant :perform, -> { ContextDev::WebWebScrapeMdParams::Action::Perform }
|
|
208
208
|
|
|
209
|
+
# Scroll the page or a selected scrollable container, waiting adaptively for content and dimensions to settle after each iteration.
|
|
210
|
+
variant :scroll, -> { ContextDev::WebWebScrapeMdParams::Action::Scroll }
|
|
211
|
+
|
|
209
212
|
class Wait < ContextDev::Internal::Type::BaseModel
|
|
210
213
|
# @!attribute do_
|
|
211
214
|
#
|
|
@@ -242,8 +245,101 @@ module ContextDev
|
|
|
242
245
|
# @param do_ [Symbol, :perform]
|
|
243
246
|
end
|
|
244
247
|
|
|
248
|
+
class Scroll < ContextDev::Internal::Type::BaseModel
|
|
249
|
+
# @!attribute do_
|
|
250
|
+
#
|
|
251
|
+
# @return [Symbol, :scroll]
|
|
252
|
+
required :do_, const: :scroll, api_name: :do
|
|
253
|
+
|
|
254
|
+
# @!attribute amount
|
|
255
|
+
# Pixels per scroll, one visible viewport, or the current scroll boundary.
|
|
256
|
+
# Defaults to viewport.
|
|
257
|
+
#
|
|
258
|
+
# @return [Integer, Symbol, ContextDev::Models::WebWebScrapeMdParams::Action::Scroll::Amount, nil]
|
|
259
|
+
optional :amount, union: -> { ContextDev::WebWebScrapeMdParams::Action::Scroll::Amount }
|
|
260
|
+
|
|
261
|
+
# @!attribute container
|
|
262
|
+
# CSS selector for the first matching scroll container. Defaults to the page.
|
|
263
|
+
#
|
|
264
|
+
# @return [String, nil]
|
|
265
|
+
optional :container, String
|
|
266
|
+
|
|
267
|
+
# @!attribute direction
|
|
268
|
+
# Direction to scroll. Defaults to down.
|
|
269
|
+
#
|
|
270
|
+
# @return [Symbol, ContextDev::Models::WebWebScrapeMdParams::Action::Scroll::Direction, nil]
|
|
271
|
+
optional :direction, enum: -> { ContextDev::WebWebScrapeMdParams::Action::Scroll::Direction }
|
|
272
|
+
|
|
273
|
+
# @!attribute max_scrolls
|
|
274
|
+
# Maximum scroll iterations. Stops early when scrolling and scrollable extent stop
|
|
275
|
+
# changing. Defaults to 1.
|
|
276
|
+
#
|
|
277
|
+
# @return [Integer, nil]
|
|
278
|
+
optional :max_scrolls, Integer, api_name: :maxScrolls
|
|
279
|
+
|
|
280
|
+
# @!method initialize(amount: nil, container: nil, direction: nil, max_scrolls: nil, do_: :scroll)
|
|
281
|
+
# Some parameter documentations has been truncated, see
|
|
282
|
+
# {ContextDev::Models::WebWebScrapeMdParams::Action::Scroll} for more details.
|
|
283
|
+
#
|
|
284
|
+
# Scroll the page or a selected scrollable container, waiting adaptively for
|
|
285
|
+
# content and dimensions to settle after each iteration.
|
|
286
|
+
#
|
|
287
|
+
# @param amount [Integer, Symbol, ContextDev::Models::WebWebScrapeMdParams::Action::Scroll::Amount] Pixels per scroll, one visible viewport, or the current scroll boundary. Default
|
|
288
|
+
#
|
|
289
|
+
# @param container [String] CSS selector for the first matching scroll container. Defaults to the page.
|
|
290
|
+
#
|
|
291
|
+
# @param direction [Symbol, ContextDev::Models::WebWebScrapeMdParams::Action::Scroll::Direction] Direction to scroll. Defaults to down.
|
|
292
|
+
#
|
|
293
|
+
# @param max_scrolls [Integer] Maximum scroll iterations. Stops early when scrolling and scrollable extent stop
|
|
294
|
+
#
|
|
295
|
+
# @param do_ [Symbol, :scroll]
|
|
296
|
+
|
|
297
|
+
# Pixels per scroll, one visible viewport, or the current scroll boundary.
|
|
298
|
+
# Defaults to viewport.
|
|
299
|
+
#
|
|
300
|
+
# @see ContextDev::Models::WebWebScrapeMdParams::Action::Scroll#amount
|
|
301
|
+
module Amount
|
|
302
|
+
extend ContextDev::Internal::Type::Union
|
|
303
|
+
|
|
304
|
+
variant Integer
|
|
305
|
+
|
|
306
|
+
variant const: -> { ContextDev::Models::WebWebScrapeMdParams::Action::Scroll::Amount::VIEWPORT }
|
|
307
|
+
|
|
308
|
+
variant const: -> { ContextDev::Models::WebWebScrapeMdParams::Action::Scroll::Amount::MAX }
|
|
309
|
+
|
|
310
|
+
# @!method self.variants
|
|
311
|
+
# @return [Array(Integer, Symbol)]
|
|
312
|
+
|
|
313
|
+
define_sorbet_constant!(:Variants) do
|
|
314
|
+
T.type_alias { T.any(Integer, ContextDev::WebWebScrapeMdParams::Action::Scroll::Amount::TaggedSymbol) }
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
# @!group
|
|
318
|
+
|
|
319
|
+
VIEWPORT = :viewport
|
|
320
|
+
MAX = :max
|
|
321
|
+
|
|
322
|
+
# @!endgroup
|
|
323
|
+
end
|
|
324
|
+
|
|
325
|
+
# Direction to scroll. Defaults to down.
|
|
326
|
+
#
|
|
327
|
+
# @see ContextDev::Models::WebWebScrapeMdParams::Action::Scroll#direction
|
|
328
|
+
module Direction
|
|
329
|
+
extend ContextDev::Internal::Type::Enum
|
|
330
|
+
|
|
331
|
+
UP = :up
|
|
332
|
+
DOWN = :down
|
|
333
|
+
LEFT = :left
|
|
334
|
+
RIGHT = :right
|
|
335
|
+
|
|
336
|
+
# @!method self.values
|
|
337
|
+
# @return [Array<Symbol>]
|
|
338
|
+
end
|
|
339
|
+
end
|
|
340
|
+
|
|
245
341
|
# @!method self.variants
|
|
246
|
-
# @return [Array(ContextDev::Models::WebWebScrapeMdParams::Action::Wait, ContextDev::Models::WebWebScrapeMdParams::Action::Perform)]
|
|
342
|
+
# @return [Array(ContextDev::Models::WebWebScrapeMdParams::Action::Wait, ContextDev::Models::WebWebScrapeMdParams::Action::Perform, ContextDev::Models::WebWebScrapeMdParams::Action::Scroll)]
|
|
247
343
|
end
|
|
248
344
|
|
|
249
345
|
# Fetch the target page through a residential proxy in this country (ISO 3166-1
|
|
@@ -9,12 +9,14 @@ module ContextDev
|
|
|
9
9
|
# Crawl a website, use the provided JSON Schema and instructions to prioritize
|
|
10
10
|
# relevant internal links, and extract structured data from the selected pages.
|
|
11
11
|
#
|
|
12
|
-
# @overload extract(schema:, url:, fact_check: nil, follow_subdomains: nil, include_frames: nil, instructions: nil, max_age_ms: nil, max_depth: nil, max_pages: nil, pdf: nil, settle_animations: nil, stop_after_ms: nil, tags: nil, timeout_ms: nil, wait_for_ms: nil, request_options: {})
|
|
12
|
+
# @overload extract(schema:, url:, actions: nil, fact_check: nil, follow_subdomains: nil, include_frames: nil, instructions: nil, max_age_ms: nil, max_depth: nil, max_pages: nil, pdf: nil, settle_animations: nil, stop_after_ms: nil, tags: nil, timeout_ms: nil, wait_for_ms: nil, request_options: {})
|
|
13
13
|
#
|
|
14
|
-
# @param schema [Hash{Symbol=>Object}] JSON Schema for the returned data object.
|
|
14
|
+
# @param schema [Hash{Symbol=>Object}] JSON Schema for the returned data object. Image fields such as `image_urls` or `
|
|
15
15
|
#
|
|
16
16
|
# @param url [String] The starting website URL to crawl and extract from. Must include http:// or http
|
|
17
17
|
#
|
|
18
|
+
# @param actions [Array<ContextDev::Models::WebExtractParams::Action::Wait, ContextDev::Models::WebExtractParams::Action::Perform, ContextDev::Models::WebExtractParams::Action::Scroll>] Optional browser actions executed in order on the requested page after it loads,
|
|
19
|
+
#
|
|
18
20
|
# @param fact_check [Boolean] When true, every returned value must be grounded in facts stated on the page; fi
|
|
19
21
|
#
|
|
20
22
|
# @param follow_subdomains [Boolean] When true, follow links on subdomains of the starting URL's domain.
|
|
@@ -320,11 +322,11 @@ module ContextDev
|
|
|
320
322
|
#
|
|
321
323
|
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
322
324
|
#
|
|
323
|
-
# @param url_regex [String] Regex pattern. Only URLs matching this pattern will be followed and scraped.
|
|
325
|
+
# @param url_regex [String] Regex pattern. Only URLs matching this pattern will be followed and scraped. An
|
|
324
326
|
#
|
|
325
327
|
# @param use_main_content_only [Boolean] Extract only the main content, stripping headers, footers, sidebars, and navigat
|
|
326
328
|
#
|
|
327
|
-
# @param wait_for_ms [Integer]
|
|
329
|
+
# @param wait_for_ms [Integer] Browser wait time in milliseconds after initial page load for each crawled page.
|
|
328
330
|
#
|
|
329
331
|
# @param zdr [Symbol, ContextDev::Models::WebWebCrawlMdParams::Zdr] Set to enabled to bypass shared caches and omit request and response content fro
|
|
330
332
|
#
|
|
@@ -354,7 +356,7 @@ module ContextDev
|
|
|
354
356
|
#
|
|
355
357
|
# @param url [String] Full URL to scrape (must include http:// or https:// protocol)
|
|
356
358
|
#
|
|
357
|
-
# @param actions [Array<ContextDev::Models::WebWebScrapeHTMLParams::Action::Wait, ContextDev::Models::WebWebScrapeHTMLParams::Action::Perform>, nil] Optional browser actions executed in array order after the page loads and before
|
|
359
|
+
# @param actions [Array<ContextDev::Models::WebWebScrapeHTMLParams::Action::Wait, ContextDev::Models::WebWebScrapeHTMLParams::Action::Perform, ContextDev::Models::WebWebScrapeHTMLParams::Action::Scroll>, nil] Optional browser actions executed in array order after the page loads and before
|
|
358
360
|
#
|
|
359
361
|
# @param country [Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al
|
|
360
362
|
#
|
|
@@ -421,7 +423,7 @@ module ContextDev
|
|
|
421
423
|
#
|
|
422
424
|
# @param url [String] Page URL to inspect. Must include http:// or https://.
|
|
423
425
|
#
|
|
424
|
-
# @param actions [Array<ContextDev::Models::WebWebScrapeImagesParams::Action::Wait, ContextDev::Models::WebWebScrapeImagesParams::Action::Perform>, nil] Optional browser actions executed in array order after the page loads and before
|
|
426
|
+
# @param actions [Array<ContextDev::Models::WebWebScrapeImagesParams::Action::Wait, ContextDev::Models::WebWebScrapeImagesParams::Action::Perform, ContextDev::Models::WebWebScrapeImagesParams::Action::Scroll>, nil] Optional browser actions executed in array order after the page loads and before
|
|
425
427
|
#
|
|
426
428
|
# @param dedupe [Boolean] When true, visually duplicate images are removed: every image is loaded and perc
|
|
427
429
|
#
|
|
@@ -495,7 +497,7 @@ module ContextDev
|
|
|
495
497
|
#
|
|
496
498
|
# @param url [String] Full URL to scrape into LLM usable Markdown (must include http:// or https:// pr
|
|
497
499
|
#
|
|
498
|
-
# @param actions [Array<ContextDev::Models::WebWebScrapeMdParams::Action::Wait, ContextDev::Models::WebWebScrapeMdParams::Action::Perform>, nil] Optional browser actions executed in array order after the page loads and before
|
|
500
|
+
# @param actions [Array<ContextDev::Models::WebWebScrapeMdParams::Action::Wait, ContextDev::Models::WebWebScrapeMdParams::Action::Perform, ContextDev::Models::WebWebScrapeMdParams::Action::Scroll>, nil] Optional browser actions executed in array order after the page loads and before
|
|
499
501
|
#
|
|
500
502
|
# @param country [Symbol, ContextDev::Models::WebWebScrapeMdParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al
|
|
501
503
|
#
|
data/lib/context_dev/version.rb
CHANGED