context.dev 2.4.0 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +16 -0
- data/README.md +1 -1
- data/lib/context_dev/models/ai_extract_product_params.rb +9 -1
- data/lib/context_dev/models/ai_extract_products_params.rb +18 -2
- data/lib/context_dev/models/brand_retrieve_params.rb +109 -22
- data/lib/context_dev/models/brand_retrieve_response.rb +4 -6
- data/lib/context_dev/models/brand_retrieve_simplified_params.rb +32 -3
- data/lib/context_dev/models/industry_retrieve_naics_params.rb +11 -1
- data/lib/context_dev/models/industry_retrieve_sic_params.rb +11 -1
- data/lib/context_dev/models/monitor_create_params.rb +6 -5
- data/lib/context_dev/models/monitor_create_response.rb +6 -5
- data/lib/context_dev/models/monitor_get_credit_usage_params.rb +30 -0
- data/lib/context_dev/models/monitor_get_credit_usage_response.rb +59 -0
- data/lib/context_dev/models/monitor_get_limits_params.rb +14 -0
- data/lib/context_dev/models/monitor_get_limits_response.rb +52 -0
- data/lib/context_dev/models/monitor_list_account_changes_params.rb +16 -7
- data/lib/context_dev/models/monitor_list_account_changes_response.rb +2 -1
- data/lib/context_dev/models/monitor_list_account_runs_params.rb +7 -10
- data/lib/context_dev/models/monitor_list_changes_params.rb +8 -4
- data/lib/context_dev/models/monitor_list_changes_response.rb +2 -1
- data/lib/context_dev/models/monitor_list_params.rb +18 -19
- data/lib/context_dev/models/monitor_list_response.rb +6 -5
- data/lib/context_dev/models/monitor_list_runs_params.rb +7 -10
- data/lib/context_dev/models/monitor_retrieve_change_response.rb +2 -1
- data/lib/context_dev/models/monitor_retrieve_response.rb +6 -5
- data/lib/context_dev/models/monitor_update_params.rb +6 -5
- data/lib/context_dev/models/monitor_update_response.rb +6 -5
- data/lib/context_dev/models/parse_handle_params.rb +199 -33
- data/lib/context_dev/models/utility_prefetch_params.rb +9 -1
- data/lib/context_dev/models/web_extract_competitors_params.rb +11 -1
- data/lib/context_dev/models/web_extract_fonts_params.rb +14 -4
- data/lib/context_dev/models/web_extract_params.rb +19 -1
- data/lib/context_dev/models/web_extract_styleguide_params.rb +14 -4
- data/lib/context_dev/models/web_screenshot_params.rb +69 -18
- data/lib/context_dev/models/web_screenshot_response.rb +3 -2
- data/lib/context_dev/models/web_search_params.rb +9 -1
- data/lib/context_dev/models/web_web_crawl_md_params.rb +34 -1
- data/lib/context_dev/models/web_web_scrape_html_params.rb +268 -29
- data/lib/context_dev/models/web_web_scrape_images_params.rb +208 -19
- data/lib/context_dev/models/web_web_scrape_md_params.rb +352 -38
- data/lib/context_dev/models/web_web_scrape_sitemap_params.rb +45 -1
- data/lib/context_dev/models.rb +4 -0
- data/lib/context_dev/resources/ai.rb +3 -1
- data/lib/context_dev/resources/brand.rb +6 -2
- data/lib/context_dev/resources/industry.rb +6 -2
- data/lib/context_dev/resources/monitors.rb +70 -32
- data/lib/context_dev/resources/parse.rb +15 -11
- data/lib/context_dev/resources/utility.rb +3 -1
- data/lib/context_dev/resources/web.rb +104 -55
- data/lib/context_dev/version.rb +1 -1
- data/lib/context_dev.rb +4 -0
- data/rbi/context_dev/models/ai_extract_product_params.rbi +11 -0
- data/rbi/context_dev/models/ai_extract_products_params.rbi +22 -0
- data/rbi/context_dev/models/brand_retrieve_params.rbi +175 -64
- data/rbi/context_dev/models/brand_retrieve_response.rbi +8 -11
- data/rbi/context_dev/models/brand_retrieve_simplified_params.rbi +68 -5
- data/rbi/context_dev/models/industry_retrieve_naics_params.rbi +15 -0
- data/rbi/context_dev/models/industry_retrieve_sic_params.rbi +15 -0
- data/rbi/context_dev/models/monitor_create_params.rbi +6 -4
- data/rbi/context_dev/models/monitor_create_response.rbi +6 -4
- data/rbi/context_dev/models/monitor_get_credit_usage_params.rbi +60 -0
- data/rbi/context_dev/models/monitor_get_credit_usage_response.rbi +111 -0
- data/rbi/context_dev/models/monitor_get_limits_params.rbi +30 -0
- data/rbi/context_dev/models/monitor_get_limits_response.rbi +105 -0
- data/rbi/context_dev/models/monitor_list_account_changes_params.rbi +16 -0
- data/rbi/context_dev/models/monitor_list_account_changes_response.rbi +2 -0
- data/rbi/context_dev/models/monitor_list_account_runs_params.rbi +7 -6
- data/rbi/context_dev/models/monitor_list_changes_params.rbi +8 -0
- data/rbi/context_dev/models/monitor_list_changes_response.rbi +2 -0
- data/rbi/context_dev/models/monitor_list_params.rbi +23 -31
- data/rbi/context_dev/models/monitor_list_response.rbi +6 -4
- data/rbi/context_dev/models/monitor_list_runs_params.rbi +7 -6
- data/rbi/context_dev/models/monitor_retrieve_change_response.rbi +2 -0
- data/rbi/context_dev/models/monitor_retrieve_response.rbi +6 -4
- data/rbi/context_dev/models/monitor_update_params.rbi +6 -4
- data/rbi/context_dev/models/monitor_update_response.rbi +6 -4
- data/rbi/context_dev/models/parse_handle_params.rbi +400 -46
- data/rbi/context_dev/models/utility_prefetch_params.rbi +11 -0
- data/rbi/context_dev/models/web_extract_competitors_params.rbi +15 -0
- data/rbi/context_dev/models/web_extract_fonts_params.rbi +19 -7
- data/rbi/context_dev/models/web_extract_params.rbi +26 -0
- data/rbi/context_dev/models/web_extract_styleguide_params.rbi +19 -7
- data/rbi/context_dev/models/web_screenshot_params.rbi +108 -37
- data/rbi/context_dev/models/web_screenshot_response.rbi +4 -2
- data/rbi/context_dev/models/web_search_params.rbi +11 -0
- data/rbi/context_dev/models/web_web_crawl_md_params.rbi +53 -0
- data/rbi/context_dev/models/web_web_scrape_html_params.rbi +554 -49
- data/rbi/context_dev/models/web_web_scrape_images_params.rbi +426 -30
- data/rbi/context_dev/models/web_web_scrape_md_params.rbi +742 -61
- data/rbi/context_dev/models/web_web_scrape_sitemap_params.rbi +82 -0
- data/rbi/context_dev/models.rbi +4 -0
- data/rbi/context_dev/resources/ai.rbi +3 -0
- data/rbi/context_dev/resources/brand.rbi +9 -1
- data/rbi/context_dev/resources/industry.rbi +10 -0
- data/rbi/context_dev/resources/monitors.rbi +55 -11
- data/rbi/context_dev/resources/parse.rbi +46 -19
- data/rbi/context_dev/resources/utility.rbi +3 -0
- data/rbi/context_dev/resources/web.rbi +224 -48
- data/sig/context_dev/models/ai_extract_product_params.rbs +12 -1
- data/sig/context_dev/models/ai_extract_products_params.rbs +14 -0
- data/sig/context_dev/models/brand_retrieve_params.rbs +97 -51
- data/sig/context_dev/models/brand_retrieve_response.rbs +2 -2
- data/sig/context_dev/models/brand_retrieve_simplified_params.rbs +34 -5
- data/sig/context_dev/models/industry_retrieve_naics_params.rbs +7 -0
- data/sig/context_dev/models/industry_retrieve_sic_params.rbs +7 -0
- data/sig/context_dev/models/monitor_get_credit_usage_params.rbs +32 -0
- data/sig/context_dev/models/monitor_get_credit_usage_response.rbs +52 -0
- data/sig/context_dev/models/monitor_get_limits_params.rbs +15 -0
- data/sig/context_dev/models/monitor_get_limits_response.rbs +43 -0
- data/sig/context_dev/models/monitor_list_params.rbs +8 -14
- data/sig/context_dev/models/parse_handle_params.rbs +124 -25
- data/sig/context_dev/models/utility_prefetch_params.rbs +7 -0
- data/sig/context_dev/models/web_extract_competitors_params.rbs +12 -1
- data/sig/context_dev/models/web_extract_fonts_params.rbs +10 -5
- data/sig/context_dev/models/web_extract_params.rbs +14 -0
- data/sig/context_dev/models/web_extract_styleguide_params.rbs +10 -5
- data/sig/context_dev/models/web_screenshot_params.rbs +41 -20
- data/sig/context_dev/models/web_search_params.rbs +7 -0
- data/sig/context_dev/models/web_web_crawl_md_params.rbs +28 -1
- data/sig/context_dev/models/web_web_scrape_html_params.rbs +175 -47
- data/sig/context_dev/models/web_web_scrape_images_params.rbs +128 -38
- data/sig/context_dev/models/web_web_scrape_md_params.rbs +229 -62
- data/sig/context_dev/models/web_web_scrape_sitemap_params.rbs +35 -1
- data/sig/context_dev/models.rbs +4 -0
- data/sig/context_dev/resources/ai.rbs +1 -0
- data/sig/context_dev/resources/brand.rbs +3 -1
- data/sig/context_dev/resources/industry.rbs +2 -0
- data/sig/context_dev/resources/monitors.rbs +12 -2
- data/sig/context_dev/resources/parse.rbs +8 -5
- data/sig/context_dev/resources/utility.rbs +1 -0
- data/sig/context_dev/resources/web.rbs +47 -26
- metadata +14 -2
|
@@ -18,8 +18,26 @@ module ContextDev
|
|
|
18
18
|
sig { returns(String) }
|
|
19
19
|
attr_accessor :url
|
|
20
20
|
|
|
21
|
-
#
|
|
22
|
-
#
|
|
21
|
+
# Optional browser actions executed in array order after the page loads and before
|
|
22
|
+
# content is captured. Requires a paid plan. Send a JSON array in the query
|
|
23
|
+
# parameter. Maximum: 5 actions.
|
|
24
|
+
sig do
|
|
25
|
+
returns(
|
|
26
|
+
T.nilable(
|
|
27
|
+
T::Array[
|
|
28
|
+
T.any(
|
|
29
|
+
ContextDev::WebWebScrapeHTMLParams::Action::Wait,
|
|
30
|
+
ContextDev::WebWebScrapeHTMLParams::Action::Perform
|
|
31
|
+
)
|
|
32
|
+
]
|
|
33
|
+
)
|
|
34
|
+
)
|
|
35
|
+
end
|
|
36
|
+
attr_accessor :actions
|
|
37
|
+
|
|
38
|
+
# Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
|
|
39
|
+
# residential proxy exit location. Must be one of Context.dev's supported
|
|
40
|
+
# countries. When provided, Context.dev fetches the target page from that country.
|
|
23
41
|
sig do
|
|
24
42
|
returns(
|
|
25
43
|
T.nilable(ContextDev::WebWebScrapeHTMLParams::Country::OrSymbol)
|
|
@@ -38,10 +56,7 @@ module ContextDev
|
|
|
38
56
|
# Exclusion takes precedence: an element matching both is removed. Examples:
|
|
39
57
|
# "nav", "footer", ".ad-banner", "[aria-hidden=true]".
|
|
40
58
|
sig { returns(T.nilable(T::Array[String])) }
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
sig { params(exclude_selectors: T::Array[String]).void }
|
|
44
|
-
attr_writer :exclude_selectors
|
|
59
|
+
attr_accessor :exclude_selectors
|
|
45
60
|
|
|
46
61
|
# Optional outbound HTTP headers forwarded only to the target URL, sent as
|
|
47
62
|
# deep-object query params such as headers[X-Custom]=value. When provided, caching
|
|
@@ -53,29 +68,40 @@ module ContextDev
|
|
|
53
68
|
attr_writer :headers
|
|
54
69
|
|
|
55
70
|
# When true, iframes are rendered inline into the returned HTML.
|
|
56
|
-
sig
|
|
71
|
+
sig do
|
|
72
|
+
returns(
|
|
73
|
+
T.nilable(
|
|
74
|
+
T.any(
|
|
75
|
+
T::Boolean,
|
|
76
|
+
ContextDev::WebWebScrapeHTMLParams::IncludeFrames::OrSymbol
|
|
77
|
+
)
|
|
78
|
+
)
|
|
79
|
+
)
|
|
80
|
+
end
|
|
57
81
|
attr_reader :include_frames
|
|
58
82
|
|
|
59
|
-
sig
|
|
83
|
+
sig do
|
|
84
|
+
params(
|
|
85
|
+
include_frames:
|
|
86
|
+
T.any(
|
|
87
|
+
T::Boolean,
|
|
88
|
+
ContextDev::WebWebScrapeHTMLParams::IncludeFrames::OrSymbol
|
|
89
|
+
)
|
|
90
|
+
).void
|
|
91
|
+
end
|
|
60
92
|
attr_writer :include_frames
|
|
61
93
|
|
|
62
94
|
# CSS selectors. When provided, only matching subtrees (and their descendants) are
|
|
63
95
|
# kept and everything else is dropped. When omitted, the entire document is kept.
|
|
64
96
|
# Examples: "article.main", "#content", "[role=main]".
|
|
65
97
|
sig { returns(T.nilable(T::Array[String])) }
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
sig { params(include_selectors: T::Array[String]).void }
|
|
69
|
-
attr_writer :include_selectors
|
|
98
|
+
attr_accessor :include_selectors
|
|
70
99
|
|
|
71
100
|
# Return a cached result if a prior scrape for the same parameters exists and is
|
|
72
101
|
# younger than this many milliseconds. Defaults to 1 day (86400000 ms) when
|
|
73
102
|
# omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh.
|
|
74
103
|
sig { returns(T.nilable(Integer)) }
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
sig { params(max_age_ms: Integer).void }
|
|
78
|
-
attr_writer :max_age_ms
|
|
104
|
+
attr_accessor :max_age_ms
|
|
79
105
|
|
|
80
106
|
# PDF parsing controls. Use start/end to limit text extraction and embedded-image
|
|
81
107
|
# detection/OCR to an inclusive 1-based page range.
|
|
@@ -88,12 +114,38 @@ module ContextDev
|
|
|
88
114
|
# When true, waits briefly for CSS and transition animations to settle before
|
|
89
115
|
# extracting HTML. Defaults to false. This adds a bit of latency in exchange for
|
|
90
116
|
# more stable output on animated pages.
|
|
91
|
-
sig
|
|
117
|
+
sig do
|
|
118
|
+
returns(
|
|
119
|
+
T.nilable(
|
|
120
|
+
T.any(
|
|
121
|
+
T::Boolean,
|
|
122
|
+
ContextDev::WebWebScrapeHTMLParams::SettleAnimations::OrSymbol
|
|
123
|
+
)
|
|
124
|
+
)
|
|
125
|
+
)
|
|
126
|
+
end
|
|
92
127
|
attr_reader :settle_animations
|
|
93
128
|
|
|
94
|
-
sig
|
|
129
|
+
sig do
|
|
130
|
+
params(
|
|
131
|
+
settle_animations:
|
|
132
|
+
T.any(
|
|
133
|
+
T::Boolean,
|
|
134
|
+
ContextDev::WebWebScrapeHTMLParams::SettleAnimations::OrSymbol
|
|
135
|
+
)
|
|
136
|
+
).void
|
|
137
|
+
end
|
|
95
138
|
attr_writer :settle_animations
|
|
96
139
|
|
|
140
|
+
# Optional comma-separated caller-defined tags for tracking this request. Tags are
|
|
141
|
+
# recorded on the request's usage log and can be used to filter usage on the
|
|
142
|
+
# dashboard usage page. Up to 20 tags, each 1-50 characters.
|
|
143
|
+
sig { returns(T.nilable(T::Array[String])) }
|
|
144
|
+
attr_reader :tags
|
|
145
|
+
|
|
146
|
+
sig { params(tags: T::Array[String]).void }
|
|
147
|
+
attr_writer :tags
|
|
148
|
+
|
|
97
149
|
# Optional timeout in milliseconds for the request. If the request takes longer
|
|
98
150
|
# than this value, it will be aborted with a 408 status code. Maximum allowed
|
|
99
151
|
# value is 300000ms (5 minutes).
|
|
@@ -105,42 +157,98 @@ module ContextDev
|
|
|
105
157
|
|
|
106
158
|
# When true, return only the page's main content in the HTML response, excluding
|
|
107
159
|
# headers, footers, sidebars, and navigation when detectable.
|
|
108
|
-
sig
|
|
160
|
+
sig do
|
|
161
|
+
returns(
|
|
162
|
+
T.nilable(
|
|
163
|
+
T.any(
|
|
164
|
+
T::Boolean,
|
|
165
|
+
ContextDev::WebWebScrapeHTMLParams::UseMainContentOnly::OrSymbol
|
|
166
|
+
)
|
|
167
|
+
)
|
|
168
|
+
)
|
|
169
|
+
end
|
|
109
170
|
attr_reader :use_main_content_only
|
|
110
171
|
|
|
111
|
-
sig
|
|
172
|
+
sig do
|
|
173
|
+
params(
|
|
174
|
+
use_main_content_only:
|
|
175
|
+
T.any(
|
|
176
|
+
T::Boolean,
|
|
177
|
+
ContextDev::WebWebScrapeHTMLParams::UseMainContentOnly::OrSymbol
|
|
178
|
+
)
|
|
179
|
+
).void
|
|
180
|
+
end
|
|
112
181
|
attr_writer :use_main_content_only
|
|
113
182
|
|
|
114
183
|
# Optional browser wait time in milliseconds after initial page load. Min: 0. Max:
|
|
115
184
|
# 30000 (30 seconds).
|
|
116
185
|
sig { returns(T.nilable(Integer)) }
|
|
117
|
-
|
|
186
|
+
attr_accessor :wait_for_ms
|
|
118
187
|
|
|
119
|
-
|
|
120
|
-
|
|
188
|
+
# Set to enabled to bypass shared caches and omit request and response content
|
|
189
|
+
# from retained usage logs. Requires zero data retention to be enabled for your
|
|
190
|
+
# organization (contact support@context.dev), otherwise the request fails with
|
|
191
|
+
# ZDR_NOT_ENABLED. Successful ZDR responses include X-Context-ZDR: true.
|
|
192
|
+
sig do
|
|
193
|
+
returns(T.nilable(ContextDev::WebWebScrapeHTMLParams::Zdr::OrSymbol))
|
|
194
|
+
end
|
|
195
|
+
attr_reader :zdr
|
|
196
|
+
|
|
197
|
+
sig do
|
|
198
|
+
params(zdr: ContextDev::WebWebScrapeHTMLParams::Zdr::OrSymbol).void
|
|
199
|
+
end
|
|
200
|
+
attr_writer :zdr
|
|
121
201
|
|
|
122
202
|
sig do
|
|
123
203
|
params(
|
|
124
204
|
url: String,
|
|
205
|
+
actions:
|
|
206
|
+
T.nilable(
|
|
207
|
+
T::Array[
|
|
208
|
+
T.any(
|
|
209
|
+
ContextDev::WebWebScrapeHTMLParams::Action::Wait::OrHash,
|
|
210
|
+
ContextDev::WebWebScrapeHTMLParams::Action::Perform::OrHash
|
|
211
|
+
)
|
|
212
|
+
]
|
|
213
|
+
),
|
|
125
214
|
country: ContextDev::WebWebScrapeHTMLParams::Country::OrSymbol,
|
|
126
|
-
exclude_selectors: T::Array[String],
|
|
215
|
+
exclude_selectors: T.nilable(T::Array[String]),
|
|
127
216
|
headers: T::Hash[Symbol, String],
|
|
128
|
-
include_frames:
|
|
129
|
-
|
|
130
|
-
|
|
217
|
+
include_frames:
|
|
218
|
+
T.any(
|
|
219
|
+
T::Boolean,
|
|
220
|
+
ContextDev::WebWebScrapeHTMLParams::IncludeFrames::OrSymbol
|
|
221
|
+
),
|
|
222
|
+
include_selectors: T.nilable(T::Array[String]),
|
|
223
|
+
max_age_ms: T.nilable(Integer),
|
|
131
224
|
pdf: ContextDev::WebWebScrapeHTMLParams::Pdf::OrHash,
|
|
132
|
-
settle_animations:
|
|
225
|
+
settle_animations:
|
|
226
|
+
T.any(
|
|
227
|
+
T::Boolean,
|
|
228
|
+
ContextDev::WebWebScrapeHTMLParams::SettleAnimations::OrSymbol
|
|
229
|
+
),
|
|
230
|
+
tags: T::Array[String],
|
|
133
231
|
timeout_ms: Integer,
|
|
134
|
-
use_main_content_only:
|
|
135
|
-
|
|
232
|
+
use_main_content_only:
|
|
233
|
+
T.any(
|
|
234
|
+
T::Boolean,
|
|
235
|
+
ContextDev::WebWebScrapeHTMLParams::UseMainContentOnly::OrSymbol
|
|
236
|
+
),
|
|
237
|
+
wait_for_ms: T.nilable(Integer),
|
|
238
|
+
zdr: ContextDev::WebWebScrapeHTMLParams::Zdr::OrSymbol,
|
|
136
239
|
request_options: ContextDev::RequestOptions::OrHash
|
|
137
240
|
).returns(T.attached_class)
|
|
138
241
|
end
|
|
139
242
|
def self.new(
|
|
140
243
|
# Full URL to scrape (must include http:// or https:// protocol)
|
|
141
244
|
url:,
|
|
142
|
-
#
|
|
143
|
-
#
|
|
245
|
+
# Optional browser actions executed in array order after the page loads and before
|
|
246
|
+
# content is captured. Requires a paid plan. Send a JSON array in the query
|
|
247
|
+
# parameter. Maximum: 5 actions.
|
|
248
|
+
actions: nil,
|
|
249
|
+
# Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
|
|
250
|
+
# residential proxy exit location. Must be one of Context.dev's supported
|
|
251
|
+
# countries. When provided, Context.dev fetches the target page from that country.
|
|
144
252
|
country: nil,
|
|
145
253
|
# CSS selectors to remove from the result. Applied after includeSelectors.
|
|
146
254
|
# Exclusion takes precedence: an element matching both is removed. Examples:
|
|
@@ -167,6 +275,10 @@ module ContextDev
|
|
|
167
275
|
# extracting HTML. Defaults to false. This adds a bit of latency in exchange for
|
|
168
276
|
# more stable output on animated pages.
|
|
169
277
|
settle_animations: nil,
|
|
278
|
+
# Optional comma-separated caller-defined tags for tracking this request. Tags are
|
|
279
|
+
# recorded on the request's usage log and can be used to filter usage on the
|
|
280
|
+
# dashboard usage page. Up to 20 tags, each 1-50 characters.
|
|
281
|
+
tags: nil,
|
|
170
282
|
# Optional timeout in milliseconds for the request. If the request takes longer
|
|
171
283
|
# than this value, it will be aborted with a 408 status code. Maximum allowed
|
|
172
284
|
# value is 300000ms (5 minutes).
|
|
@@ -177,6 +289,11 @@ module ContextDev
|
|
|
177
289
|
# Optional browser wait time in milliseconds after initial page load. Min: 0. Max:
|
|
178
290
|
# 30000 (30 seconds).
|
|
179
291
|
wait_for_ms: nil,
|
|
292
|
+
# Set to enabled to bypass shared caches and omit request and response content
|
|
293
|
+
# from retained usage logs. Requires zero data retention to be enabled for your
|
|
294
|
+
# organization (contact support@context.dev), otherwise the request fails with
|
|
295
|
+
# ZDR_NOT_ENABLED. Successful ZDR responses include X-Context-ZDR: true.
|
|
296
|
+
zdr: nil,
|
|
180
297
|
request_options: {}
|
|
181
298
|
)
|
|
182
299
|
end
|
|
@@ -185,17 +302,40 @@ module ContextDev
|
|
|
185
302
|
override.returns(
|
|
186
303
|
{
|
|
187
304
|
url: String,
|
|
305
|
+
actions:
|
|
306
|
+
T.nilable(
|
|
307
|
+
T::Array[
|
|
308
|
+
T.any(
|
|
309
|
+
ContextDev::WebWebScrapeHTMLParams::Action::Wait,
|
|
310
|
+
ContextDev::WebWebScrapeHTMLParams::Action::Perform
|
|
311
|
+
)
|
|
312
|
+
]
|
|
313
|
+
),
|
|
188
314
|
country: ContextDev::WebWebScrapeHTMLParams::Country::OrSymbol,
|
|
189
|
-
exclude_selectors: T::Array[String],
|
|
315
|
+
exclude_selectors: T.nilable(T::Array[String]),
|
|
190
316
|
headers: T::Hash[Symbol, String],
|
|
191
|
-
include_frames:
|
|
192
|
-
|
|
193
|
-
|
|
317
|
+
include_frames:
|
|
318
|
+
T.any(
|
|
319
|
+
T::Boolean,
|
|
320
|
+
ContextDev::WebWebScrapeHTMLParams::IncludeFrames::OrSymbol
|
|
321
|
+
),
|
|
322
|
+
include_selectors: T.nilable(T::Array[String]),
|
|
323
|
+
max_age_ms: T.nilable(Integer),
|
|
194
324
|
pdf: ContextDev::WebWebScrapeHTMLParams::Pdf,
|
|
195
|
-
settle_animations:
|
|
325
|
+
settle_animations:
|
|
326
|
+
T.any(
|
|
327
|
+
T::Boolean,
|
|
328
|
+
ContextDev::WebWebScrapeHTMLParams::SettleAnimations::OrSymbol
|
|
329
|
+
),
|
|
330
|
+
tags: T::Array[String],
|
|
196
331
|
timeout_ms: Integer,
|
|
197
|
-
use_main_content_only:
|
|
198
|
-
|
|
332
|
+
use_main_content_only:
|
|
333
|
+
T.any(
|
|
334
|
+
T::Boolean,
|
|
335
|
+
ContextDev::WebWebScrapeHTMLParams::UseMainContentOnly::OrSymbol
|
|
336
|
+
),
|
|
337
|
+
wait_for_ms: T.nilable(Integer),
|
|
338
|
+
zdr: ContextDev::WebWebScrapeHTMLParams::Zdr::OrSymbol,
|
|
199
339
|
request_options: ContextDev::RequestOptions
|
|
200
340
|
}
|
|
201
341
|
)
|
|
@@ -203,8 +343,83 @@ module ContextDev
|
|
|
203
343
|
def to_hash
|
|
204
344
|
end
|
|
205
345
|
|
|
206
|
-
#
|
|
207
|
-
#
|
|
346
|
+
# Browser action discriminated by `do`. Each variant exposes only its applicable
|
|
347
|
+
# fields.
|
|
348
|
+
module Action
|
|
349
|
+
extend ContextDev::Internal::Type::Union
|
|
350
|
+
|
|
351
|
+
Variants =
|
|
352
|
+
T.type_alias do
|
|
353
|
+
T.any(
|
|
354
|
+
ContextDev::WebWebScrapeHTMLParams::Action::Wait,
|
|
355
|
+
ContextDev::WebWebScrapeHTMLParams::Action::Perform
|
|
356
|
+
)
|
|
357
|
+
end
|
|
358
|
+
|
|
359
|
+
class Wait < ContextDev::Internal::Type::BaseModel
|
|
360
|
+
OrHash =
|
|
361
|
+
T.type_alias do
|
|
362
|
+
T.any(
|
|
363
|
+
ContextDev::WebWebScrapeHTMLParams::Action::Wait,
|
|
364
|
+
ContextDev::Internal::AnyHash
|
|
365
|
+
)
|
|
366
|
+
end
|
|
367
|
+
|
|
368
|
+
sig { returns(Symbol) }
|
|
369
|
+
attr_accessor :do_
|
|
370
|
+
|
|
371
|
+
sig { returns(Integer) }
|
|
372
|
+
attr_accessor :time_ms
|
|
373
|
+
|
|
374
|
+
# Pause for a fixed number of milliseconds before continuing to the next action.
|
|
375
|
+
sig do
|
|
376
|
+
params(time_ms: Integer, do_: Symbol).returns(T.attached_class)
|
|
377
|
+
end
|
|
378
|
+
def self.new(time_ms:, do_: :wait)
|
|
379
|
+
end
|
|
380
|
+
|
|
381
|
+
sig { override.returns({ do_: Symbol, time_ms: Integer }) }
|
|
382
|
+
def to_hash
|
|
383
|
+
end
|
|
384
|
+
end
|
|
385
|
+
|
|
386
|
+
class Perform < ContextDev::Internal::Type::BaseModel
|
|
387
|
+
OrHash =
|
|
388
|
+
T.type_alias do
|
|
389
|
+
T.any(
|
|
390
|
+
ContextDev::WebWebScrapeHTMLParams::Action::Perform,
|
|
391
|
+
ContextDev::Internal::AnyHash
|
|
392
|
+
)
|
|
393
|
+
end
|
|
394
|
+
|
|
395
|
+
sig { returns(String) }
|
|
396
|
+
attr_accessor :action
|
|
397
|
+
|
|
398
|
+
sig { returns(Symbol) }
|
|
399
|
+
attr_accessor :do_
|
|
400
|
+
|
|
401
|
+
# Resolve and perform one natural-language browser action.
|
|
402
|
+
sig { params(action: String, do_: Symbol).returns(T.attached_class) }
|
|
403
|
+
def self.new(action:, do_: :perform)
|
|
404
|
+
end
|
|
405
|
+
|
|
406
|
+
sig { override.returns({ action: String, do_: Symbol }) }
|
|
407
|
+
def to_hash
|
|
408
|
+
end
|
|
409
|
+
end
|
|
410
|
+
|
|
411
|
+
sig do
|
|
412
|
+
override.returns(
|
|
413
|
+
T::Array[ContextDev::WebWebScrapeHTMLParams::Action::Variants]
|
|
414
|
+
)
|
|
415
|
+
end
|
|
416
|
+
def self.variants
|
|
417
|
+
end
|
|
418
|
+
end
|
|
419
|
+
|
|
420
|
+
# Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
|
|
421
|
+
# residential proxy exit location. Must be one of Context.dev's supported
|
|
422
|
+
# countries. When provided, Context.dev fetches the target page from that country.
|
|
208
423
|
module Country
|
|
209
424
|
extend ContextDev::Internal::Type::Enum
|
|
210
425
|
|
|
@@ -632,6 +847,46 @@ module ContextDev
|
|
|
632
847
|
end
|
|
633
848
|
end
|
|
634
849
|
|
|
850
|
+
# When true, iframes are rendered inline into the returned HTML.
|
|
851
|
+
module IncludeFrames
|
|
852
|
+
extend ContextDev::Internal::Type::Union
|
|
853
|
+
|
|
854
|
+
Variants =
|
|
855
|
+
T.type_alias do
|
|
856
|
+
T.any(
|
|
857
|
+
T::Boolean,
|
|
858
|
+
ContextDev::WebWebScrapeHTMLParams::IncludeFrames::TaggedSymbol
|
|
859
|
+
)
|
|
860
|
+
end
|
|
861
|
+
|
|
862
|
+
sig do
|
|
863
|
+
override.returns(
|
|
864
|
+
T::Array[
|
|
865
|
+
ContextDev::WebWebScrapeHTMLParams::IncludeFrames::Variants
|
|
866
|
+
]
|
|
867
|
+
)
|
|
868
|
+
end
|
|
869
|
+
def self.variants
|
|
870
|
+
end
|
|
871
|
+
|
|
872
|
+
TaggedSymbol =
|
|
873
|
+
T.type_alias do
|
|
874
|
+
T.all(Symbol, ContextDev::WebWebScrapeHTMLParams::IncludeFrames)
|
|
875
|
+
end
|
|
876
|
+
OrSymbol = T.type_alias { T.any(Symbol, String) }
|
|
877
|
+
|
|
878
|
+
TRUE =
|
|
879
|
+
T.let(
|
|
880
|
+
:true,
|
|
881
|
+
ContextDev::WebWebScrapeHTMLParams::IncludeFrames::TaggedSymbol
|
|
882
|
+
)
|
|
883
|
+
FALSE =
|
|
884
|
+
T.let(
|
|
885
|
+
:false,
|
|
886
|
+
ContextDev::WebWebScrapeHTMLParams::IncludeFrames::TaggedSymbol
|
|
887
|
+
)
|
|
888
|
+
end
|
|
889
|
+
|
|
635
890
|
class Pdf < ContextDev::Internal::Type::BaseModel
|
|
636
891
|
OrHash =
|
|
637
892
|
T.type_alias do
|
|
@@ -652,18 +907,52 @@ module ContextDev
|
|
|
652
907
|
# When true, detect and OCR images embedded in the selected PDF pages, inserting
|
|
653
908
|
# recognized text at each image's position in page reading order while preserving
|
|
654
909
|
# the PDF text layer. This is separate from automatic scanned-PDF OCR fallback.
|
|
655
|
-
sig
|
|
910
|
+
sig do
|
|
911
|
+
returns(
|
|
912
|
+
T.nilable(
|
|
913
|
+
T.any(
|
|
914
|
+
T::Boolean,
|
|
915
|
+
ContextDev::WebWebScrapeHTMLParams::Pdf::Ocr::OrSymbol
|
|
916
|
+
)
|
|
917
|
+
)
|
|
918
|
+
)
|
|
919
|
+
end
|
|
656
920
|
attr_reader :ocr
|
|
657
921
|
|
|
658
|
-
sig
|
|
922
|
+
sig do
|
|
923
|
+
params(
|
|
924
|
+
ocr:
|
|
925
|
+
T.any(
|
|
926
|
+
T::Boolean,
|
|
927
|
+
ContextDev::WebWebScrapeHTMLParams::Pdf::Ocr::OrSymbol
|
|
928
|
+
)
|
|
929
|
+
).void
|
|
930
|
+
end
|
|
659
931
|
attr_writer :ocr
|
|
660
932
|
|
|
661
933
|
# When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and
|
|
662
934
|
# a 400 WEBSITE_ACCESS_ERROR is returned.
|
|
663
|
-
sig
|
|
935
|
+
sig do
|
|
936
|
+
returns(
|
|
937
|
+
T.nilable(
|
|
938
|
+
T.any(
|
|
939
|
+
T::Boolean,
|
|
940
|
+
ContextDev::WebWebScrapeHTMLParams::Pdf::ShouldParse::OrSymbol
|
|
941
|
+
)
|
|
942
|
+
)
|
|
943
|
+
)
|
|
944
|
+
end
|
|
664
945
|
attr_reader :should_parse
|
|
665
946
|
|
|
666
|
-
sig
|
|
947
|
+
sig do
|
|
948
|
+
params(
|
|
949
|
+
should_parse:
|
|
950
|
+
T.any(
|
|
951
|
+
T::Boolean,
|
|
952
|
+
ContextDev::WebWebScrapeHTMLParams::Pdf::ShouldParse::OrSymbol
|
|
953
|
+
)
|
|
954
|
+
).void
|
|
955
|
+
end
|
|
667
956
|
attr_writer :should_parse
|
|
668
957
|
|
|
669
958
|
# First 1-based PDF page to parse. When omitted, parsing starts at the first page.
|
|
@@ -678,8 +967,16 @@ module ContextDev
|
|
|
678
967
|
sig do
|
|
679
968
|
params(
|
|
680
969
|
end_: Integer,
|
|
681
|
-
ocr:
|
|
682
|
-
|
|
970
|
+
ocr:
|
|
971
|
+
T.any(
|
|
972
|
+
T::Boolean,
|
|
973
|
+
ContextDev::WebWebScrapeHTMLParams::Pdf::Ocr::OrSymbol
|
|
974
|
+
),
|
|
975
|
+
should_parse:
|
|
976
|
+
T.any(
|
|
977
|
+
T::Boolean,
|
|
978
|
+
ContextDev::WebWebScrapeHTMLParams::Pdf::ShouldParse::OrSymbol
|
|
979
|
+
),
|
|
683
980
|
start: Integer
|
|
684
981
|
).returns(T.attached_class)
|
|
685
982
|
end
|
|
@@ -703,14 +1000,222 @@ module ContextDev
|
|
|
703
1000
|
override.returns(
|
|
704
1001
|
{
|
|
705
1002
|
end_: Integer,
|
|
706
|
-
ocr:
|
|
707
|
-
|
|
1003
|
+
ocr:
|
|
1004
|
+
T.any(
|
|
1005
|
+
T::Boolean,
|
|
1006
|
+
ContextDev::WebWebScrapeHTMLParams::Pdf::Ocr::OrSymbol
|
|
1007
|
+
),
|
|
1008
|
+
should_parse:
|
|
1009
|
+
T.any(
|
|
1010
|
+
T::Boolean,
|
|
1011
|
+
ContextDev::WebWebScrapeHTMLParams::Pdf::ShouldParse::OrSymbol
|
|
1012
|
+
),
|
|
708
1013
|
start: Integer
|
|
709
1014
|
}
|
|
710
1015
|
)
|
|
711
1016
|
end
|
|
712
1017
|
def to_hash
|
|
713
1018
|
end
|
|
1019
|
+
|
|
1020
|
+
# When true, detect and OCR images embedded in the selected PDF pages, inserting
|
|
1021
|
+
# recognized text at each image's position in page reading order while preserving
|
|
1022
|
+
# the PDF text layer. This is separate from automatic scanned-PDF OCR fallback.
|
|
1023
|
+
module Ocr
|
|
1024
|
+
extend ContextDev::Internal::Type::Union
|
|
1025
|
+
|
|
1026
|
+
Variants =
|
|
1027
|
+
T.type_alias do
|
|
1028
|
+
T.any(
|
|
1029
|
+
T::Boolean,
|
|
1030
|
+
ContextDev::WebWebScrapeHTMLParams::Pdf::Ocr::TaggedSymbol
|
|
1031
|
+
)
|
|
1032
|
+
end
|
|
1033
|
+
|
|
1034
|
+
sig do
|
|
1035
|
+
override.returns(
|
|
1036
|
+
T::Array[ContextDev::WebWebScrapeHTMLParams::Pdf::Ocr::Variants]
|
|
1037
|
+
)
|
|
1038
|
+
end
|
|
1039
|
+
def self.variants
|
|
1040
|
+
end
|
|
1041
|
+
|
|
1042
|
+
TaggedSymbol =
|
|
1043
|
+
T.type_alias do
|
|
1044
|
+
T.all(Symbol, ContextDev::WebWebScrapeHTMLParams::Pdf::Ocr)
|
|
1045
|
+
end
|
|
1046
|
+
OrSymbol = T.type_alias { T.any(Symbol, String) }
|
|
1047
|
+
|
|
1048
|
+
TRUE =
|
|
1049
|
+
T.let(
|
|
1050
|
+
:true,
|
|
1051
|
+
ContextDev::WebWebScrapeHTMLParams::Pdf::Ocr::TaggedSymbol
|
|
1052
|
+
)
|
|
1053
|
+
FALSE =
|
|
1054
|
+
T.let(
|
|
1055
|
+
:false,
|
|
1056
|
+
ContextDev::WebWebScrapeHTMLParams::Pdf::Ocr::TaggedSymbol
|
|
1057
|
+
)
|
|
1058
|
+
end
|
|
1059
|
+
|
|
1060
|
+
# When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and
|
|
1061
|
+
# a 400 WEBSITE_ACCESS_ERROR is returned.
|
|
1062
|
+
module ShouldParse
|
|
1063
|
+
extend ContextDev::Internal::Type::Union
|
|
1064
|
+
|
|
1065
|
+
Variants =
|
|
1066
|
+
T.type_alias do
|
|
1067
|
+
T.any(
|
|
1068
|
+
T::Boolean,
|
|
1069
|
+
ContextDev::WebWebScrapeHTMLParams::Pdf::ShouldParse::TaggedSymbol
|
|
1070
|
+
)
|
|
1071
|
+
end
|
|
1072
|
+
|
|
1073
|
+
sig do
|
|
1074
|
+
override.returns(
|
|
1075
|
+
T::Array[
|
|
1076
|
+
ContextDev::WebWebScrapeHTMLParams::Pdf::ShouldParse::Variants
|
|
1077
|
+
]
|
|
1078
|
+
)
|
|
1079
|
+
end
|
|
1080
|
+
def self.variants
|
|
1081
|
+
end
|
|
1082
|
+
|
|
1083
|
+
TaggedSymbol =
|
|
1084
|
+
T.type_alias do
|
|
1085
|
+
T.all(
|
|
1086
|
+
Symbol,
|
|
1087
|
+
ContextDev::WebWebScrapeHTMLParams::Pdf::ShouldParse
|
|
1088
|
+
)
|
|
1089
|
+
end
|
|
1090
|
+
OrSymbol = T.type_alias { T.any(Symbol, String) }
|
|
1091
|
+
|
|
1092
|
+
TRUE =
|
|
1093
|
+
T.let(
|
|
1094
|
+
:true,
|
|
1095
|
+
ContextDev::WebWebScrapeHTMLParams::Pdf::ShouldParse::TaggedSymbol
|
|
1096
|
+
)
|
|
1097
|
+
FALSE =
|
|
1098
|
+
T.let(
|
|
1099
|
+
:false,
|
|
1100
|
+
ContextDev::WebWebScrapeHTMLParams::Pdf::ShouldParse::TaggedSymbol
|
|
1101
|
+
)
|
|
1102
|
+
end
|
|
1103
|
+
end
|
|
1104
|
+
|
|
1105
|
+
# When true, waits briefly for CSS and transition animations to settle before
|
|
1106
|
+
# extracting HTML. Defaults to false. This adds a bit of latency in exchange for
|
|
1107
|
+
# more stable output on animated pages.
|
|
1108
|
+
module SettleAnimations
|
|
1109
|
+
extend ContextDev::Internal::Type::Union
|
|
1110
|
+
|
|
1111
|
+
Variants =
|
|
1112
|
+
T.type_alias do
|
|
1113
|
+
T.any(
|
|
1114
|
+
T::Boolean,
|
|
1115
|
+
ContextDev::WebWebScrapeHTMLParams::SettleAnimations::TaggedSymbol
|
|
1116
|
+
)
|
|
1117
|
+
end
|
|
1118
|
+
|
|
1119
|
+
sig do
|
|
1120
|
+
override.returns(
|
|
1121
|
+
T::Array[
|
|
1122
|
+
ContextDev::WebWebScrapeHTMLParams::SettleAnimations::Variants
|
|
1123
|
+
]
|
|
1124
|
+
)
|
|
1125
|
+
end
|
|
1126
|
+
def self.variants
|
|
1127
|
+
end
|
|
1128
|
+
|
|
1129
|
+
TaggedSymbol =
|
|
1130
|
+
T.type_alias do
|
|
1131
|
+
T.all(Symbol, ContextDev::WebWebScrapeHTMLParams::SettleAnimations)
|
|
1132
|
+
end
|
|
1133
|
+
OrSymbol = T.type_alias { T.any(Symbol, String) }
|
|
1134
|
+
|
|
1135
|
+
TRUE =
|
|
1136
|
+
T.let(
|
|
1137
|
+
:true,
|
|
1138
|
+
ContextDev::WebWebScrapeHTMLParams::SettleAnimations::TaggedSymbol
|
|
1139
|
+
)
|
|
1140
|
+
FALSE =
|
|
1141
|
+
T.let(
|
|
1142
|
+
:false,
|
|
1143
|
+
ContextDev::WebWebScrapeHTMLParams::SettleAnimations::TaggedSymbol
|
|
1144
|
+
)
|
|
1145
|
+
end
|
|
1146
|
+
|
|
1147
|
+
# When true, return only the page's main content in the HTML response, excluding
|
|
1148
|
+
# headers, footers, sidebars, and navigation when detectable.
|
|
1149
|
+
module UseMainContentOnly
|
|
1150
|
+
extend ContextDev::Internal::Type::Union
|
|
1151
|
+
|
|
1152
|
+
Variants =
|
|
1153
|
+
T.type_alias do
|
|
1154
|
+
T.any(
|
|
1155
|
+
T::Boolean,
|
|
1156
|
+
ContextDev::WebWebScrapeHTMLParams::UseMainContentOnly::TaggedSymbol
|
|
1157
|
+
)
|
|
1158
|
+
end
|
|
1159
|
+
|
|
1160
|
+
sig do
|
|
1161
|
+
override.returns(
|
|
1162
|
+
T::Array[
|
|
1163
|
+
ContextDev::WebWebScrapeHTMLParams::UseMainContentOnly::Variants
|
|
1164
|
+
]
|
|
1165
|
+
)
|
|
1166
|
+
end
|
|
1167
|
+
def self.variants
|
|
1168
|
+
end
|
|
1169
|
+
|
|
1170
|
+
TaggedSymbol =
|
|
1171
|
+
T.type_alias do
|
|
1172
|
+
T.all(
|
|
1173
|
+
Symbol,
|
|
1174
|
+
ContextDev::WebWebScrapeHTMLParams::UseMainContentOnly
|
|
1175
|
+
)
|
|
1176
|
+
end
|
|
1177
|
+
OrSymbol = T.type_alias { T.any(Symbol, String) }
|
|
1178
|
+
|
|
1179
|
+
TRUE =
|
|
1180
|
+
T.let(
|
|
1181
|
+
:true,
|
|
1182
|
+
ContextDev::WebWebScrapeHTMLParams::UseMainContentOnly::TaggedSymbol
|
|
1183
|
+
)
|
|
1184
|
+
FALSE =
|
|
1185
|
+
T.let(
|
|
1186
|
+
:false,
|
|
1187
|
+
ContextDev::WebWebScrapeHTMLParams::UseMainContentOnly::TaggedSymbol
|
|
1188
|
+
)
|
|
1189
|
+
end
|
|
1190
|
+
|
|
1191
|
+
# Set to enabled to bypass shared caches and omit request and response content
|
|
1192
|
+
# from retained usage logs. Requires zero data retention to be enabled for your
|
|
1193
|
+
# organization (contact support@context.dev), otherwise the request fails with
|
|
1194
|
+
# ZDR_NOT_ENABLED. Successful ZDR responses include X-Context-ZDR: true.
|
|
1195
|
+
module Zdr
|
|
1196
|
+
extend ContextDev::Internal::Type::Enum
|
|
1197
|
+
|
|
1198
|
+
TaggedSymbol =
|
|
1199
|
+
T.type_alias do
|
|
1200
|
+
T.all(Symbol, ContextDev::WebWebScrapeHTMLParams::Zdr)
|
|
1201
|
+
end
|
|
1202
|
+
OrSymbol = T.type_alias { T.any(Symbol, String) }
|
|
1203
|
+
|
|
1204
|
+
ENABLED =
|
|
1205
|
+
T.let(:enabled, ContextDev::WebWebScrapeHTMLParams::Zdr::TaggedSymbol)
|
|
1206
|
+
DISABLED =
|
|
1207
|
+
T.let(
|
|
1208
|
+
:disabled,
|
|
1209
|
+
ContextDev::WebWebScrapeHTMLParams::Zdr::TaggedSymbol
|
|
1210
|
+
)
|
|
1211
|
+
|
|
1212
|
+
sig do
|
|
1213
|
+
override.returns(
|
|
1214
|
+
T::Array[ContextDev::WebWebScrapeHTMLParams::Zdr::TaggedSymbol]
|
|
1215
|
+
)
|
|
1216
|
+
end
|
|
1217
|
+
def self.values
|
|
1218
|
+
end
|
|
714
1219
|
end
|
|
715
1220
|
end
|
|
716
1221
|
end
|