context.dev 2.8.0 → 2.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +28 -0
- data/README.md +1 -1
- data/lib/context_dev/client.rb +10 -0
- data/lib/context_dev/models/batch_delete_params.rb +22 -0
- data/lib/context_dev/models/batch_delete_response.rb +60 -0
- data/lib/context_dev/models/batch_get_results_response.rb +46 -4
- data/lib/context_dev/models/batch_list_response.rb +13 -4
- data/lib/context_dev/models/batch_retrieve_response.rb +13 -4
- data/lib/context_dev/models/batch_submit_params.rb +2080 -26
- data/lib/context_dev/models/batch_submit_response.rb +125 -531
- data/lib/context_dev/models/brand_retrieve_response.rb +50 -1
- data/lib/context_dev/models/brand_search_params.rb +41 -3
- data/lib/context_dev/models/brand_search_response.rb +3 -2
- data/lib/context_dev/models/crawl_controls.rb +21 -15
- data/lib/context_dev/models/news_search_params.rb +467 -0
- data/lib/context_dev/models/news_search_response.rb +238 -0
- data/lib/context_dev/models/parse_handle_params.rb +20 -147
- data/lib/context_dev/models/person_enrich_params.rb +176 -0
- data/lib/context_dev/models/person_enrich_response.rb +641 -0
- data/lib/context_dev/models/utility_prefetch_params.rb +19 -16
- data/lib/context_dev/models/utility_prefetch_response.rb +4 -5
- data/lib/context_dev/models/web_screenshot_params.rb +3 -30
- data/lib/context_dev/models/web_search_response.rb +1 -0
- data/lib/context_dev/models/web_web_crawl_md_params.rb +5 -4
- data/lib/context_dev/models/web_web_crawl_md_response.rb +30 -1
- data/lib/context_dev/models/web_web_scrape_html_params.rb +20 -156
- data/lib/context_dev/models/web_web_scrape_html_response.rb +30 -1
- data/lib/context_dev/models/web_web_scrape_images_params.rb +12 -124
- data/lib/context_dev/models/web_web_scrape_md_params.rb +40 -241
- data/lib/context_dev/models/web_web_scrape_md_response.rb +41 -2
- data/lib/context_dev/models/web_web_scrape_sitemap_params.rb +11 -1
- data/lib/context_dev/models/web_web_scrape_sitemap_response.rb +3 -2
- data/lib/context_dev/models.rb +6 -0
- data/lib/context_dev/resources/batch.rb +33 -7
- data/lib/context_dev/resources/brand.rb +10 -10
- data/lib/context_dev/resources/news.rb +51 -0
- data/lib/context_dev/resources/parse.rb +5 -5
- data/lib/context_dev/resources/people.rb +56 -0
- data/lib/context_dev/resources/utility.rb +7 -6
- data/lib/context_dev/resources/web.rb +46 -24
- data/lib/context_dev/version.rb +1 -1
- data/lib/context_dev.rb +8 -0
- data/rbi/context_dev/client.rbi +8 -0
- data/rbi/context_dev/models/batch_delete_params.rbi +40 -0
- data/rbi/context_dev/models/batch_delete_response.rbi +116 -0
- data/rbi/context_dev/models/batch_get_results_response.rbi +85 -3
- data/rbi/context_dev/models/batch_list_response.rbi +24 -8
- data/rbi/context_dev/models/batch_retrieve_response.rbi +24 -8
- data/rbi/context_dev/models/batch_submit_params.rbi +6421 -44
- data/rbi/context_dev/models/batch_submit_response.rbi +184 -1151
- data/rbi/context_dev/models/brand_retrieve_response.rbi +152 -0
- data/rbi/context_dev/models/brand_search_params.rbi +71 -2
- data/rbi/context_dev/models/brand_search_response.rbi +4 -2
- data/rbi/context_dev/models/crawl_controls.rbi +22 -28
- data/rbi/context_dev/models/news_search_params.rbi +1294 -0
- data/rbi/context_dev/models/news_search_response.rbi +423 -0
- data/rbi/context_dev/models/parse_handle_params.rbi +30 -323
- data/rbi/context_dev/models/person_enrich_params.rbi +332 -0
- data/rbi/context_dev/models/person_enrich_response.rbi +1295 -0
- data/rbi/context_dev/models/utility_prefetch_params.rbi +24 -16
- data/rbi/context_dev/models/utility_prefetch_response.rbi +8 -6
- data/rbi/context_dev/models/web_screenshot_params.rbi +4 -71
- data/rbi/context_dev/models/web_search_response.rbi +5 -0
- data/rbi/context_dev/models/web_web_crawl_md_params.rbi +8 -6
- data/rbi/context_dev/models/web_web_crawl_md_response.rbi +67 -0
- data/rbi/context_dev/models/web_web_scrape_html_params.rbi +30 -363
- data/rbi/context_dev/models/web_web_scrape_html_response.rbi +65 -0
- data/rbi/context_dev/models/web_web_scrape_images_params.rbi +16 -287
- data/rbi/context_dev/models/web_web_scrape_md_params.rbi +57 -558
- data/rbi/context_dev/models/web_web_scrape_md_response.rbi +80 -0
- data/rbi/context_dev/models/web_web_scrape_sitemap_params.rbi +15 -0
- data/rbi/context_dev/models/web_web_scrape_sitemap_response.rbi +4 -2
- data/rbi/context_dev/models.rbi +6 -0
- data/rbi/context_dev/resources/batch.rbi +32 -10
- data/rbi/context_dev/resources/brand.rbi +14 -8
- data/rbi/context_dev/resources/news.rbi +46 -0
- data/rbi/context_dev/resources/parse.rbi +10 -26
- data/rbi/context_dev/resources/people.rbi +47 -0
- data/rbi/context_dev/resources/utility.rbi +8 -6
- data/rbi/context_dev/resources/web.rbi +49 -66
- data/sig/context_dev/client.rbs +4 -0
- data/sig/context_dev/models/batch_delete_params.rbs +23 -0
- data/sig/context_dev/models/batch_delete_response.rbs +57 -0
- data/sig/context_dev/models/batch_get_results_response.rbs +30 -2
- data/sig/context_dev/models/batch_list_response.rbs +16 -2
- data/sig/context_dev/models/batch_retrieve_response.rbs +16 -2
- data/sig/context_dev/models/batch_submit_params.rbs +2666 -15
- data/sig/context_dev/models/batch_submit_response.rbs +78 -466
- data/sig/context_dev/models/brand_retrieve_response.rbs +62 -0
- data/sig/context_dev/models/brand_search_params.rbs +38 -1
- data/sig/context_dev/models/crawl_controls.rbs +16 -16
- data/sig/context_dev/models/news_search_params.rbs +532 -0
- data/sig/context_dev/models/news_search_response.rbs +206 -0
- data/sig/context_dev/models/parse_handle_params.rbs +25 -90
- data/sig/context_dev/models/person_enrich_params.rbs +199 -0
- data/sig/context_dev/models/person_enrich_response.rbs +638 -0
- data/sig/context_dev/models/utility_prefetch_params.rbs +2 -1
- data/sig/context_dev/models/utility_prefetch_response.rbs +2 -1
- data/sig/context_dev/models/web_screenshot_params.rbs +5 -18
- data/sig/context_dev/models/web_search_response.rbs +2 -0
- data/sig/context_dev/models/web_web_crawl_md_response.rbs +21 -0
- data/sig/context_dev/models/web_web_scrape_html_params.rbs +24 -94
- data/sig/context_dev/models/web_web_scrape_html_response.rbs +21 -0
- data/sig/context_dev/models/web_web_scrape_images_params.rbs +20 -72
- data/sig/context_dev/models/web_web_scrape_md_params.rbs +46 -148
- data/sig/context_dev/models/web_web_scrape_md_response.rbs +28 -0
- data/sig/context_dev/models/web_web_scrape_sitemap_params.rbs +7 -0
- data/sig/context_dev/models.rbs +6 -0
- data/sig/context_dev/resources/batch.rbs +8 -2
- data/sig/context_dev/resources/brand.rbs +3 -0
- data/sig/context_dev/resources/news.rbs +17 -0
- data/sig/context_dev/resources/parse.rbs +5 -5
- data/sig/context_dev/resources/people.rbs +19 -0
- data/sig/context_dev/resources/web.rbs +13 -11
- metadata +26 -2
|
@@ -8,13 +8,14 @@ module ContextDev
|
|
|
8
8
|
include ContextDev::Internal::Type::RequestParameters
|
|
9
9
|
|
|
10
10
|
# @!attribute identifier
|
|
11
|
-
# Identifier of the
|
|
11
|
+
# Identifier of the target to prefetch. Provide exactly one of domain or email.
|
|
12
12
|
#
|
|
13
13
|
# @return [ContextDev::Models::UtilityPrefetchParams::Identifier::UtilityPrefetchDomainIdentifier, ContextDev::Models::UtilityPrefetchParams::Identifier::UtilityPrefetchEmailIdentifier]
|
|
14
14
|
required :identifier, union: -> { ContextDev::UtilityPrefetchParams::Identifier }
|
|
15
15
|
|
|
16
16
|
# @!attribute type
|
|
17
|
-
# What to prefetch
|
|
17
|
+
# What to prefetch: 'brand' warms the brand data cache, 'styleguide' warms the
|
|
18
|
+
# styleguide cache.
|
|
18
19
|
#
|
|
19
20
|
# @return [Symbol, ContextDev::Models::UtilityPrefetchParams::Type]
|
|
20
21
|
required :type, enum: -> { ContextDev::UtilityPrefetchParams::Type }
|
|
@@ -37,9 +38,9 @@ module ContextDev
|
|
|
37
38
|
# Some parameter documentations has been truncated, see
|
|
38
39
|
# {ContextDev::Models::UtilityPrefetchParams} for more details.
|
|
39
40
|
#
|
|
40
|
-
# @param identifier [ContextDev::Models::UtilityPrefetchParams::Identifier::UtilityPrefetchDomainIdentifier, ContextDev::Models::UtilityPrefetchParams::Identifier::UtilityPrefetchEmailIdentifier] Identifier of the
|
|
41
|
+
# @param identifier [ContextDev::Models::UtilityPrefetchParams::Identifier::UtilityPrefetchDomainIdentifier, ContextDev::Models::UtilityPrefetchParams::Identifier::UtilityPrefetchEmailIdentifier] Identifier of the target to prefetch. Provide exactly one of domain or email.
|
|
41
42
|
#
|
|
42
|
-
# @param type [Symbol, ContextDev::Models::UtilityPrefetchParams::Type] What to prefetch
|
|
43
|
+
# @param type [Symbol, ContextDev::Models::UtilityPrefetchParams::Type] What to prefetch: 'brand' warms the brand data cache, 'styleguide' warms the sty
|
|
43
44
|
#
|
|
44
45
|
# @param tags [Array<String>] Optional tags for tracking usage. Up to 20 tags, each 1 to 50 characters.
|
|
45
46
|
#
|
|
@@ -47,34 +48,34 @@ module ContextDev
|
|
|
47
48
|
#
|
|
48
49
|
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}]
|
|
49
50
|
|
|
50
|
-
# Identifier of the
|
|
51
|
+
# Identifier of the target to prefetch. Provide exactly one of domain or email.
|
|
51
52
|
module Identifier
|
|
52
53
|
extend ContextDev::Internal::Type::Union
|
|
53
54
|
|
|
54
|
-
# Prefetch
|
|
55
|
+
# Prefetch by domain.
|
|
55
56
|
variant -> { ContextDev::UtilityPrefetchParams::Identifier::UtilityPrefetchDomainIdentifier }
|
|
56
57
|
|
|
57
|
-
# Prefetch
|
|
58
|
+
# Prefetch by email. The domain will be extracted and validated.
|
|
58
59
|
variant -> { ContextDev::UtilityPrefetchParams::Identifier::UtilityPrefetchEmailIdentifier }
|
|
59
60
|
|
|
60
61
|
class UtilityPrefetchDomainIdentifier < ContextDev::Internal::Type::BaseModel
|
|
61
62
|
# @!attribute domain
|
|
62
|
-
# Domain name to prefetch
|
|
63
|
+
# Domain name to prefetch data for
|
|
63
64
|
#
|
|
64
65
|
# @return [String]
|
|
65
66
|
required :domain, String
|
|
66
67
|
|
|
67
68
|
# @!method initialize(domain:)
|
|
68
|
-
# Prefetch
|
|
69
|
+
# Prefetch by domain.
|
|
69
70
|
#
|
|
70
|
-
# @param domain [String] Domain name to prefetch
|
|
71
|
+
# @param domain [String] Domain name to prefetch data for
|
|
71
72
|
end
|
|
72
73
|
|
|
73
74
|
class UtilityPrefetchEmailIdentifier < ContextDev::Internal::Type::BaseModel
|
|
74
75
|
# @!attribute email
|
|
75
|
-
# Email address to prefetch
|
|
76
|
-
#
|
|
77
|
-
#
|
|
76
|
+
# Email address to prefetch data for. The domain will be extracted from the email.
|
|
77
|
+
# Free email providers (gmail.com, yahoo.com, etc.) and disposable email addresses
|
|
78
|
+
# are not allowed.
|
|
78
79
|
#
|
|
79
80
|
# @return [String]
|
|
80
81
|
required :email, String
|
|
@@ -84,20 +85,22 @@ module ContextDev
|
|
|
84
85
|
# {ContextDev::Models::UtilityPrefetchParams::Identifier::UtilityPrefetchEmailIdentifier}
|
|
85
86
|
# for more details.
|
|
86
87
|
#
|
|
87
|
-
# Prefetch
|
|
88
|
+
# Prefetch by email. The domain will be extracted and validated.
|
|
88
89
|
#
|
|
89
|
-
# @param email [String] Email address to prefetch
|
|
90
|
+
# @param email [String] Email address to prefetch data for. The domain will be extracted from the email.
|
|
90
91
|
end
|
|
91
92
|
|
|
92
93
|
# @!method self.variants
|
|
93
94
|
# @return [Array(ContextDev::Models::UtilityPrefetchParams::Identifier::UtilityPrefetchDomainIdentifier, ContextDev::Models::UtilityPrefetchParams::Identifier::UtilityPrefetchEmailIdentifier)]
|
|
94
95
|
end
|
|
95
96
|
|
|
96
|
-
# What to prefetch
|
|
97
|
+
# What to prefetch: 'brand' warms the brand data cache, 'styleguide' warms the
|
|
98
|
+
# styleguide cache.
|
|
97
99
|
module Type
|
|
98
100
|
extend ContextDev::Internal::Type::Enum
|
|
99
101
|
|
|
100
102
|
BRAND = :brand
|
|
103
|
+
STYLEGUIDE = :styleguide
|
|
101
104
|
|
|
102
105
|
# @!method self.values
|
|
103
106
|
# @return [Array<Symbol>]
|
|
@@ -30,8 +30,7 @@ module ContextDev
|
|
|
30
30
|
optional :status, String
|
|
31
31
|
|
|
32
32
|
# @!attribute type
|
|
33
|
-
# The type of prefetch that was queued, echoed from the request
|
|
34
|
-
# 'brand')
|
|
33
|
+
# The type of prefetch that was queued, echoed from the request
|
|
35
34
|
#
|
|
36
35
|
# @return [Symbol, ContextDev::Models::UtilityPrefetchResponse::Type, nil]
|
|
37
36
|
optional :type, enum: -> { ContextDev::Models::UtilityPrefetchResponse::Type }
|
|
@@ -48,7 +47,7 @@ module ContextDev
|
|
|
48
47
|
#
|
|
49
48
|
# @param status [String] Status of the response, e.g., 'ok'
|
|
50
49
|
#
|
|
51
|
-
# @param type [Symbol, ContextDev::Models::UtilityPrefetchResponse::Type] The type of prefetch that was queued, echoed from the request
|
|
50
|
+
# @param type [Symbol, ContextDev::Models::UtilityPrefetchResponse::Type] The type of prefetch that was queued, echoed from the request
|
|
52
51
|
|
|
53
52
|
# @see ContextDev::Models::UtilityPrefetchResponse#key_metadata
|
|
54
53
|
class KeyMetadata < ContextDev::Internal::Type::BaseModel
|
|
@@ -73,14 +72,14 @@ module ContextDev
|
|
|
73
72
|
# @param credits_remaining [Integer] The number of credits remaining for your organization after this request.
|
|
74
73
|
end
|
|
75
74
|
|
|
76
|
-
# The type of prefetch that was queued, echoed from the request
|
|
77
|
-
# 'brand')
|
|
75
|
+
# The type of prefetch that was queued, echoed from the request
|
|
78
76
|
#
|
|
79
77
|
# @see ContextDev::Models::UtilityPrefetchResponse#type
|
|
80
78
|
module Type
|
|
81
79
|
extend ContextDev::Internal::Type::Enum
|
|
82
80
|
|
|
83
81
|
BRAND = :brand
|
|
82
|
+
STYLEGUIDE = :styleguide
|
|
84
83
|
|
|
85
84
|
# @!method self.values
|
|
86
85
|
# @return [Array<Symbol>]
|
|
@@ -50,8 +50,8 @@ module ContextDev
|
|
|
50
50
|
# dismiss cookie banner before capture. If 'false' or not provided, captures the
|
|
51
51
|
# page without that step.
|
|
52
52
|
#
|
|
53
|
-
# @return [Boolean,
|
|
54
|
-
optional :handle_cookie_popup,
|
|
53
|
+
# @return [Boolean, nil]
|
|
54
|
+
optional :handle_cookie_popup, ContextDev::Internal::Type::Boolean
|
|
55
55
|
|
|
56
56
|
# @!attribute max_age_ms
|
|
57
57
|
# Return a cached screenshot if a prior screenshot for the same parameters exists
|
|
@@ -135,7 +135,7 @@ module ContextDev
|
|
|
135
135
|
#
|
|
136
136
|
# @param full_screenshot [Symbol, ContextDev::Models::WebScreenshotParams::FullScreenshot] Optional parameter to determine screenshot type. If 'true', takes a full page sc
|
|
137
137
|
#
|
|
138
|
-
# @param handle_cookie_popup [Boolean
|
|
138
|
+
# @param handle_cookie_popup [Boolean] Optional parameter to control cookie/consent popup handling. If 'true', we dismi
|
|
139
139
|
#
|
|
140
140
|
# @param max_age_ms [Integer, nil] Return a cached screenshot if a prior screenshot for the same parameters exists
|
|
141
141
|
#
|
|
@@ -394,33 +394,6 @@ module ContextDev
|
|
|
394
394
|
# @return [Array<Symbol>]
|
|
395
395
|
end
|
|
396
396
|
|
|
397
|
-
# Optional parameter to control cookie/consent popup handling. If 'true', we
|
|
398
|
-
# dismiss cookie banner before capture. If 'false' or not provided, captures the
|
|
399
|
-
# page without that step.
|
|
400
|
-
module HandleCookiePopup
|
|
401
|
-
extend ContextDev::Internal::Type::Union
|
|
402
|
-
|
|
403
|
-
variant ContextDev::Internal::Type::Boolean
|
|
404
|
-
|
|
405
|
-
variant const: -> { ContextDev::Models::WebScreenshotParams::HandleCookiePopup::TRUE }
|
|
406
|
-
|
|
407
|
-
variant const: -> { ContextDev::Models::WebScreenshotParams::HandleCookiePopup::FALSE }
|
|
408
|
-
|
|
409
|
-
# @!method self.variants
|
|
410
|
-
# @return [Array(Boolean, Symbol)]
|
|
411
|
-
|
|
412
|
-
define_sorbet_constant!(:Variants) do
|
|
413
|
-
T.type_alias { T.any(T::Boolean, ContextDev::WebScreenshotParams::HandleCookiePopup::TaggedSymbol) }
|
|
414
|
-
end
|
|
415
|
-
|
|
416
|
-
# @!group
|
|
417
|
-
|
|
418
|
-
TRUE = :true
|
|
419
|
-
FALSE = :false
|
|
420
|
-
|
|
421
|
-
# @!endgroup
|
|
422
|
-
end
|
|
423
|
-
|
|
424
397
|
# Optional parameter to specify which page type to screenshot. If provided, the
|
|
425
398
|
# system will scrape the domain's links and use heuristics to find the most
|
|
426
399
|
# appropriate URL for the specified page type (30 supported languages). If not
|
|
@@ -428,9 +428,10 @@ module ContextDev
|
|
|
428
428
|
optional :end_, Integer, api_name: :end
|
|
429
429
|
|
|
430
430
|
# @!attribute ocr
|
|
431
|
-
# When true,
|
|
432
|
-
#
|
|
433
|
-
#
|
|
431
|
+
# When true, OCR the selected PDF pages that have no usable text layer (scans),
|
|
432
|
+
# replacing each recovered page's text with the OCR result while pages with a real
|
|
433
|
+
# text layer keep it. Billed at 1 credit per page OCR actually recovered, on top
|
|
434
|
+
# of the base request cost.
|
|
434
435
|
#
|
|
435
436
|
# @return [Boolean, nil]
|
|
436
437
|
optional :ocr, ContextDev::Internal::Type::Boolean
|
|
@@ -457,7 +458,7 @@ module ContextDev
|
|
|
457
458
|
#
|
|
458
459
|
# @param end_ [Integer] Last 1-based PDF page to parse. When omitted, parsing ends at the last page. Mus
|
|
459
460
|
#
|
|
460
|
-
# @param ocr [Boolean] When true,
|
|
461
|
+
# @param ocr [Boolean] When true, OCR the selected PDF pages that have no usable text layer (scans), re
|
|
461
462
|
#
|
|
462
463
|
# @param should_parse [Boolean] When true, PDF pages are fetched and parsed. When false, PDF pages are skipped e
|
|
463
464
|
#
|
|
@@ -181,6 +181,14 @@ module ContextDev
|
|
|
181
181
|
# @return [String, nil]
|
|
182
182
|
optional :favicon, String
|
|
183
183
|
|
|
184
|
+
# @!attribute headings
|
|
185
|
+
# Page headings (h1–h6) in document order, extracted from the unfiltered document.
|
|
186
|
+
# Capped at the first 500 headings. Omitted when the page has none.
|
|
187
|
+
#
|
|
188
|
+
# @return [Array<ContextDev::Models::WebWebCrawlMdResponse::Result::Metadata::Heading>, nil]
|
|
189
|
+
optional :headings,
|
|
190
|
+
-> { ContextDev::Internal::Type::ArrayOf[ContextDev::Models::WebWebCrawlMdResponse::Result::Metadata::Heading] }
|
|
191
|
+
|
|
184
192
|
# @!attribute image
|
|
185
193
|
# Primary resolved preview image from Open Graph, Twitter, or image metadata.
|
|
186
194
|
#
|
|
@@ -246,7 +254,7 @@ module ContextDev
|
|
|
246
254
|
optional :twitter,
|
|
247
255
|
-> { ContextDev::Internal::Type::HashOf[union: ContextDev::Models::WebWebCrawlMdResponse::Result::Metadata::Twitter] }
|
|
248
256
|
|
|
249
|
-
# @!method initialize(crawl_depth:, final_url:, source_url:, status_code:, success:, title:, url:, additional_meta: nil, alternates: nil, author: nil, canonical_url: nil, description: nil, favicon: nil, image: nil, json_ld: nil, keywords: nil, language: nil, modified_time: nil, open_graph: nil, published_time: nil, robots: nil, site_name: nil, twitter: nil)
|
|
257
|
+
# @!method initialize(crawl_depth:, final_url:, source_url:, status_code:, success:, title:, url:, additional_meta: nil, alternates: nil, author: nil, canonical_url: nil, description: nil, favicon: nil, headings: nil, image: nil, json_ld: nil, keywords: nil, language: nil, modified_time: nil, open_graph: nil, published_time: nil, robots: nil, site_name: nil, twitter: nil)
|
|
250
258
|
# Some parameter documentations has been truncated, see
|
|
251
259
|
# {ContextDev::Models::WebWebCrawlMdResponse::Result::Metadata} for more details.
|
|
252
260
|
#
|
|
@@ -276,6 +284,8 @@ module ContextDev
|
|
|
276
284
|
#
|
|
277
285
|
# @param favicon [String] Resolved favicon URL, when present.
|
|
278
286
|
#
|
|
287
|
+
# @param headings [Array<ContextDev::Models::WebWebCrawlMdResponse::Result::Metadata::Heading>] Page headings (h1–h6) in document order, extracted from the unfiltered document.
|
|
288
|
+
#
|
|
279
289
|
# @param image [String] Primary resolved preview image from Open Graph, Twitter, or image metadata.
|
|
280
290
|
#
|
|
281
291
|
# @param json_ld [Array<Hash{Symbol=>Object}>] JSON-LD structured data blocks parsed from the page.
|
|
@@ -345,6 +355,25 @@ module ContextDev
|
|
|
345
355
|
# @param type [String] Alternate resource MIME type, when present.
|
|
346
356
|
end
|
|
347
357
|
|
|
358
|
+
class Heading < ContextDev::Internal::Type::BaseModel
|
|
359
|
+
# @!attribute level
|
|
360
|
+
# Heading level, 1–6 (from h1–h6).
|
|
361
|
+
#
|
|
362
|
+
# @return [Integer]
|
|
363
|
+
required :level, Integer
|
|
364
|
+
|
|
365
|
+
# @!attribute text
|
|
366
|
+
# Heading text with whitespace collapsed, truncated to 1000 characters.
|
|
367
|
+
#
|
|
368
|
+
# @return [String]
|
|
369
|
+
required :text, String
|
|
370
|
+
|
|
371
|
+
# @!method initialize(level:, text:)
|
|
372
|
+
# @param level [Integer] Heading level, 1–6 (from h1–h6).
|
|
373
|
+
#
|
|
374
|
+
# @param text [String] Heading text with whitespace collapsed, truncated to 1000 characters.
|
|
375
|
+
end
|
|
376
|
+
|
|
348
377
|
module OpenGraph
|
|
349
378
|
extend ContextDev::Internal::Type::Union
|
|
350
379
|
|
|
@@ -49,8 +49,8 @@ module ContextDev
|
|
|
49
49
|
# @!attribute include_frames
|
|
50
50
|
# When true, iframes are rendered inline into the returned HTML.
|
|
51
51
|
#
|
|
52
|
-
# @return [Boolean,
|
|
53
|
-
optional :include_frames,
|
|
52
|
+
# @return [Boolean, nil]
|
|
53
|
+
optional :include_frames, ContextDev::Internal::Type::Boolean
|
|
54
54
|
|
|
55
55
|
# @!attribute include_selectors
|
|
56
56
|
# CSS selectors. When provided, only matching subtrees (and their descendants) are
|
|
@@ -80,8 +80,8 @@ module ContextDev
|
|
|
80
80
|
# extracting HTML. Defaults to false. This adds a bit of latency in exchange for
|
|
81
81
|
# more stable output on animated pages.
|
|
82
82
|
#
|
|
83
|
-
# @return [Boolean,
|
|
84
|
-
optional :settle_animations,
|
|
83
|
+
# @return [Boolean, nil]
|
|
84
|
+
optional :settle_animations, ContextDev::Internal::Type::Boolean
|
|
85
85
|
|
|
86
86
|
# @!attribute tags
|
|
87
87
|
# Optional comma-separated caller-defined tags for tracking this request. Tags are
|
|
@@ -103,8 +103,8 @@ module ContextDev
|
|
|
103
103
|
# When true, return only the page's main content in the HTML response, excluding
|
|
104
104
|
# headers, footers, sidebars, and navigation when detectable.
|
|
105
105
|
#
|
|
106
|
-
# @return [Boolean,
|
|
107
|
-
optional :use_main_content_only,
|
|
106
|
+
# @return [Boolean, nil]
|
|
107
|
+
optional :use_main_content_only, ContextDev::Internal::Type::Boolean
|
|
108
108
|
|
|
109
109
|
# @!attribute wait_for_ms
|
|
110
110
|
# Optional browser wait time in milliseconds after initial page load. Min: 0. Max:
|
|
@@ -136,7 +136,7 @@ module ContextDev
|
|
|
136
136
|
#
|
|
137
137
|
# @param headers [Hash{Symbol=>String}] Optional outbound HTTP headers forwarded only to the target URL, sent as deep-ob
|
|
138
138
|
#
|
|
139
|
-
# @param include_frames [Boolean
|
|
139
|
+
# @param include_frames [Boolean] When true, iframes are rendered inline into the returned HTML.
|
|
140
140
|
#
|
|
141
141
|
# @param include_selectors [Array<String>, nil] CSS selectors. When provided, only matching subtrees (and their descendants) are
|
|
142
142
|
#
|
|
@@ -144,13 +144,13 @@ module ContextDev
|
|
|
144
144
|
#
|
|
145
145
|
# @param pdf [ContextDev::Models::WebWebScrapeHTMLParams::Pdf] PDF parsing controls. Use start/end to limit text extraction and embedded-image
|
|
146
146
|
#
|
|
147
|
-
# @param settle_animations [Boolean
|
|
147
|
+
# @param settle_animations [Boolean] When true, waits briefly for CSS and transition animations to settle before extr
|
|
148
148
|
#
|
|
149
149
|
# @param tags [Array<String>] Optional comma-separated caller-defined tags for tracking this request. Tags are
|
|
150
150
|
#
|
|
151
151
|
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
152
152
|
#
|
|
153
|
-
# @param use_main_content_only [Boolean
|
|
153
|
+
# @param use_main_content_only [Boolean] When true, return only the page's main content in the HTML response, excluding h
|
|
154
154
|
#
|
|
155
155
|
# @param wait_for_ms [Integer, nil] Optional browser wait time in milliseconds after initial page load. Min: 0. Max:
|
|
156
156
|
#
|
|
@@ -425,31 +425,6 @@ module ContextDev
|
|
|
425
425
|
# @return [Array<Symbol>]
|
|
426
426
|
end
|
|
427
427
|
|
|
428
|
-
# When true, iframes are rendered inline into the returned HTML.
|
|
429
|
-
module IncludeFrames
|
|
430
|
-
extend ContextDev::Internal::Type::Union
|
|
431
|
-
|
|
432
|
-
variant ContextDev::Internal::Type::Boolean
|
|
433
|
-
|
|
434
|
-
variant const: -> { ContextDev::Models::WebWebScrapeHTMLParams::IncludeFrames::TRUE }
|
|
435
|
-
|
|
436
|
-
variant const: -> { ContextDev::Models::WebWebScrapeHTMLParams::IncludeFrames::FALSE }
|
|
437
|
-
|
|
438
|
-
# @!method self.variants
|
|
439
|
-
# @return [Array(Boolean, Symbol)]
|
|
440
|
-
|
|
441
|
-
define_sorbet_constant!(:Variants) do
|
|
442
|
-
T.type_alias { T.any(T::Boolean, ContextDev::WebWebScrapeHTMLParams::IncludeFrames::TaggedSymbol) }
|
|
443
|
-
end
|
|
444
|
-
|
|
445
|
-
# @!group
|
|
446
|
-
|
|
447
|
-
TRUE = :true
|
|
448
|
-
FALSE = :false
|
|
449
|
-
|
|
450
|
-
# @!endgroup
|
|
451
|
-
end
|
|
452
|
-
|
|
453
428
|
class Pdf < ContextDev::Internal::Type::BaseModel
|
|
454
429
|
# @!attribute end_
|
|
455
430
|
# Last 1-based PDF page to parse. When omitted, parsing ends at the last page.
|
|
@@ -459,21 +434,20 @@ module ContextDev
|
|
|
459
434
|
optional :end_, Integer, api_name: :end
|
|
460
435
|
|
|
461
436
|
# @!attribute ocr
|
|
462
|
-
# When true,
|
|
463
|
-
#
|
|
464
|
-
#
|
|
437
|
+
# When true, OCR the selected PDF pages that have no usable text layer (scans),
|
|
438
|
+
# replacing each recovered page's text with the OCR result while pages with a real
|
|
439
|
+
# text layer keep it. Billed at 1 credit per page OCR actually recovered, on top
|
|
440
|
+
# of the base request cost. When false, no OCR runs.
|
|
465
441
|
#
|
|
466
|
-
# @return [Boolean,
|
|
467
|
-
optional :ocr,
|
|
442
|
+
# @return [Boolean, nil]
|
|
443
|
+
optional :ocr, ContextDev::Internal::Type::Boolean
|
|
468
444
|
|
|
469
445
|
# @!attribute should_parse
|
|
470
446
|
# When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and
|
|
471
|
-
# a 400
|
|
447
|
+
# a 400 PDF_SKIPPED is returned.
|
|
472
448
|
#
|
|
473
|
-
# @return [Boolean,
|
|
474
|
-
optional :should_parse,
|
|
475
|
-
union: -> { ContextDev::WebWebScrapeHTMLParams::Pdf::ShouldParse },
|
|
476
|
-
api_name: :shouldParse
|
|
449
|
+
# @return [Boolean, nil]
|
|
450
|
+
optional :should_parse, ContextDev::Internal::Type::Boolean, api_name: :shouldParse
|
|
477
451
|
|
|
478
452
|
# @!attribute start
|
|
479
453
|
# First 1-based PDF page to parse. When omitted, parsing starts at the first page.
|
|
@@ -490,121 +464,11 @@ module ContextDev
|
|
|
490
464
|
#
|
|
491
465
|
# @param end_ [Integer] Last 1-based PDF page to parse. When omitted, parsing ends at the last page. Mus
|
|
492
466
|
#
|
|
493
|
-
# @param ocr [Boolean
|
|
467
|
+
# @param ocr [Boolean] When true, OCR the selected PDF pages that have no usable text layer (scans), re
|
|
494
468
|
#
|
|
495
|
-
# @param should_parse [Boolean
|
|
469
|
+
# @param should_parse [Boolean] When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and
|
|
496
470
|
#
|
|
497
471
|
# @param start [Integer] First 1-based PDF page to parse. When omitted, parsing starts at the first page.
|
|
498
|
-
|
|
499
|
-
# When true, detect and OCR images embedded in the selected PDF pages, inserting
|
|
500
|
-
# recognized text at each image's position in page reading order while preserving
|
|
501
|
-
# the PDF text layer. This is separate from automatic scanned-PDF OCR fallback.
|
|
502
|
-
#
|
|
503
|
-
# @see ContextDev::Models::WebWebScrapeHTMLParams::Pdf#ocr
|
|
504
|
-
module Ocr
|
|
505
|
-
extend ContextDev::Internal::Type::Union
|
|
506
|
-
|
|
507
|
-
variant ContextDev::Internal::Type::Boolean
|
|
508
|
-
|
|
509
|
-
variant const: -> { ContextDev::Models::WebWebScrapeHTMLParams::Pdf::Ocr::TRUE }
|
|
510
|
-
|
|
511
|
-
variant const: -> { ContextDev::Models::WebWebScrapeHTMLParams::Pdf::Ocr::FALSE }
|
|
512
|
-
|
|
513
|
-
# @!method self.variants
|
|
514
|
-
# @return [Array(Boolean, Symbol)]
|
|
515
|
-
|
|
516
|
-
define_sorbet_constant!(:Variants) do
|
|
517
|
-
T.type_alias { T.any(T::Boolean, ContextDev::WebWebScrapeHTMLParams::Pdf::Ocr::TaggedSymbol) }
|
|
518
|
-
end
|
|
519
|
-
|
|
520
|
-
# @!group
|
|
521
|
-
|
|
522
|
-
TRUE = :true
|
|
523
|
-
FALSE = :false
|
|
524
|
-
|
|
525
|
-
# @!endgroup
|
|
526
|
-
end
|
|
527
|
-
|
|
528
|
-
# When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and
|
|
529
|
-
# a 400 WEBSITE_ACCESS_ERROR is returned.
|
|
530
|
-
#
|
|
531
|
-
# @see ContextDev::Models::WebWebScrapeHTMLParams::Pdf#should_parse
|
|
532
|
-
module ShouldParse
|
|
533
|
-
extend ContextDev::Internal::Type::Union
|
|
534
|
-
|
|
535
|
-
variant ContextDev::Internal::Type::Boolean
|
|
536
|
-
|
|
537
|
-
variant const: -> { ContextDev::Models::WebWebScrapeHTMLParams::Pdf::ShouldParse::TRUE }
|
|
538
|
-
|
|
539
|
-
variant const: -> { ContextDev::Models::WebWebScrapeHTMLParams::Pdf::ShouldParse::FALSE }
|
|
540
|
-
|
|
541
|
-
# @!method self.variants
|
|
542
|
-
# @return [Array(Boolean, Symbol)]
|
|
543
|
-
|
|
544
|
-
define_sorbet_constant!(:Variants) do
|
|
545
|
-
T.type_alias { T.any(T::Boolean, ContextDev::WebWebScrapeHTMLParams::Pdf::ShouldParse::TaggedSymbol) }
|
|
546
|
-
end
|
|
547
|
-
|
|
548
|
-
# @!group
|
|
549
|
-
|
|
550
|
-
TRUE = :true
|
|
551
|
-
FALSE = :false
|
|
552
|
-
|
|
553
|
-
# @!endgroup
|
|
554
|
-
end
|
|
555
|
-
end
|
|
556
|
-
|
|
557
|
-
# When true, waits briefly for CSS and transition animations to settle before
|
|
558
|
-
# extracting HTML. Defaults to false. This adds a bit of latency in exchange for
|
|
559
|
-
# more stable output on animated pages.
|
|
560
|
-
module SettleAnimations
|
|
561
|
-
extend ContextDev::Internal::Type::Union
|
|
562
|
-
|
|
563
|
-
variant ContextDev::Internal::Type::Boolean
|
|
564
|
-
|
|
565
|
-
variant const: -> { ContextDev::Models::WebWebScrapeHTMLParams::SettleAnimations::TRUE }
|
|
566
|
-
|
|
567
|
-
variant const: -> { ContextDev::Models::WebWebScrapeHTMLParams::SettleAnimations::FALSE }
|
|
568
|
-
|
|
569
|
-
# @!method self.variants
|
|
570
|
-
# @return [Array(Boolean, Symbol)]
|
|
571
|
-
|
|
572
|
-
define_sorbet_constant!(:Variants) do
|
|
573
|
-
T.type_alias { T.any(T::Boolean, ContextDev::WebWebScrapeHTMLParams::SettleAnimations::TaggedSymbol) }
|
|
574
|
-
end
|
|
575
|
-
|
|
576
|
-
# @!group
|
|
577
|
-
|
|
578
|
-
TRUE = :true
|
|
579
|
-
FALSE = :false
|
|
580
|
-
|
|
581
|
-
# @!endgroup
|
|
582
|
-
end
|
|
583
|
-
|
|
584
|
-
# When true, return only the page's main content in the HTML response, excluding
|
|
585
|
-
# headers, footers, sidebars, and navigation when detectable.
|
|
586
|
-
module UseMainContentOnly
|
|
587
|
-
extend ContextDev::Internal::Type::Union
|
|
588
|
-
|
|
589
|
-
variant ContextDev::Internal::Type::Boolean
|
|
590
|
-
|
|
591
|
-
variant const: -> { ContextDev::Models::WebWebScrapeHTMLParams::UseMainContentOnly::TRUE }
|
|
592
|
-
|
|
593
|
-
variant const: -> { ContextDev::Models::WebWebScrapeHTMLParams::UseMainContentOnly::FALSE }
|
|
594
|
-
|
|
595
|
-
# @!method self.variants
|
|
596
|
-
# @return [Array(Boolean, Symbol)]
|
|
597
|
-
|
|
598
|
-
define_sorbet_constant!(:Variants) do
|
|
599
|
-
T.type_alias { T.any(T::Boolean, ContextDev::WebWebScrapeHTMLParams::UseMainContentOnly::TaggedSymbol) }
|
|
600
|
-
end
|
|
601
|
-
|
|
602
|
-
# @!group
|
|
603
|
-
|
|
604
|
-
TRUE = :true
|
|
605
|
-
FALSE = :false
|
|
606
|
-
|
|
607
|
-
# @!endgroup
|
|
608
472
|
end
|
|
609
473
|
|
|
610
474
|
# Set to enabled to bypass shared caches and omit request and response content
|
|
@@ -135,6 +135,14 @@ module ContextDev
|
|
|
135
135
|
# @return [String, nil]
|
|
136
136
|
optional :favicon, String
|
|
137
137
|
|
|
138
|
+
# @!attribute headings
|
|
139
|
+
# Page headings (h1–h6) in document order, extracted from the unfiltered document.
|
|
140
|
+
# Capped at the first 500 headings. Omitted when the page has none.
|
|
141
|
+
#
|
|
142
|
+
# @return [Array<ContextDev::Models::WebWebScrapeHTMLResponse::Metadata::Heading>, nil]
|
|
143
|
+
optional :headings,
|
|
144
|
+
-> { ContextDev::Internal::Type::ArrayOf[ContextDev::Models::WebWebScrapeHTMLResponse::Metadata::Heading] }
|
|
145
|
+
|
|
138
146
|
# @!attribute image
|
|
139
147
|
# Primary resolved preview image from Open Graph, Twitter, or image metadata.
|
|
140
148
|
#
|
|
@@ -206,7 +214,7 @@ module ContextDev
|
|
|
206
214
|
optional :twitter,
|
|
207
215
|
-> { ContextDev::Internal::Type::HashOf[union: ContextDev::Models::WebWebScrapeHTMLResponse::Metadata::Twitter] }
|
|
208
216
|
|
|
209
|
-
# @!method initialize(final_url:, source_url:, additional_meta: nil, alternates: nil, author: nil, canonical_url: nil, description: nil, favicon: nil, image: nil, json_ld: nil, keywords: nil, language: nil, modified_time: nil, open_graph: nil, published_time: nil, robots: nil, site_name: nil, title: nil, twitter: nil)
|
|
217
|
+
# @!method initialize(final_url:, source_url:, additional_meta: nil, alternates: nil, author: nil, canonical_url: nil, description: nil, favicon: nil, headings: nil, image: nil, json_ld: nil, keywords: nil, language: nil, modified_time: nil, open_graph: nil, published_time: nil, robots: nil, site_name: nil, title: nil, twitter: nil)
|
|
210
218
|
# Some parameter documentations has been truncated, see
|
|
211
219
|
# {ContextDev::Models::WebWebScrapeHTMLResponse::Metadata} for more details.
|
|
212
220
|
#
|
|
@@ -228,6 +236,8 @@ module ContextDev
|
|
|
228
236
|
#
|
|
229
237
|
# @param favicon [String] Resolved favicon URL, when present.
|
|
230
238
|
#
|
|
239
|
+
# @param headings [Array<ContextDev::Models::WebWebScrapeHTMLResponse::Metadata::Heading>] Page headings (h1–h6) in document order, extracted from the unfiltered document.
|
|
240
|
+
#
|
|
231
241
|
# @param image [String] Primary resolved preview image from Open Graph, Twitter, or image metadata.
|
|
232
242
|
#
|
|
233
243
|
# @param json_ld [Array<Hash{Symbol=>Object}>] JSON-LD structured data blocks parsed from the page.
|
|
@@ -299,6 +309,25 @@ module ContextDev
|
|
|
299
309
|
# @param type [String] Alternate resource MIME type, when present.
|
|
300
310
|
end
|
|
301
311
|
|
|
312
|
+
class Heading < ContextDev::Internal::Type::BaseModel
|
|
313
|
+
# @!attribute level
|
|
314
|
+
# Heading level, 1–6 (from h1–h6).
|
|
315
|
+
#
|
|
316
|
+
# @return [Integer]
|
|
317
|
+
required :level, Integer
|
|
318
|
+
|
|
319
|
+
# @!attribute text
|
|
320
|
+
# Heading text with whitespace collapsed, truncated to 1000 characters.
|
|
321
|
+
#
|
|
322
|
+
# @return [String]
|
|
323
|
+
required :text, String
|
|
324
|
+
|
|
325
|
+
# @!method initialize(level:, text:)
|
|
326
|
+
# @param level [Integer] Heading level, 1–6 (from h1–h6).
|
|
327
|
+
#
|
|
328
|
+
# @param text [String] Heading text with whitespace collapsed, truncated to 1000 characters.
|
|
329
|
+
end
|
|
330
|
+
|
|
302
331
|
module OpenGraph
|
|
303
332
|
extend ContextDev::Internal::Type::Union
|
|
304
333
|
|