context.dev 2.9.0 → 2.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +13 -0
- data/README.md +1 -1
- data/lib/context_dev/client.rb +5 -0
- data/lib/context_dev/models/batch_get_results_response.rb +33 -3
- data/lib/context_dev/models/batch_submit_params.rb +44 -316
- data/lib/context_dev/models/brand_search_params.rb +41 -3
- data/lib/context_dev/models/news_search_params.rb +467 -0
- data/lib/context_dev/models/news_search_response.rb +238 -0
- data/lib/context_dev/models/parse_handle_params.rb +15 -144
- data/lib/context_dev/models/person_enrich_response.rb +61 -1
- data/lib/context_dev/models/utility_prefetch_params.rb +19 -16
- data/lib/context_dev/models/utility_prefetch_response.rb +4 -5
- data/lib/context_dev/models/web_screenshot_params.rb +3 -30
- data/lib/context_dev/models/web_web_crawl_md_response.rb +30 -1
- data/lib/context_dev/models/web_web_scrape_html_params.rb +15 -153
- data/lib/context_dev/models/web_web_scrape_html_response.rb +30 -1
- data/lib/context_dev/models/web_web_scrape_images_params.rb +12 -124
- data/lib/context_dev/models/web_web_scrape_md_params.rb +35 -238
- data/lib/context_dev/models/web_web_scrape_md_response.rb +41 -2
- data/lib/context_dev/models.rb +2 -0
- data/lib/context_dev/resources/brand.rb +10 -11
- data/lib/context_dev/resources/news.rb +51 -0
- data/lib/context_dev/resources/parse.rb +5 -5
- data/lib/context_dev/resources/utility.rb +7 -6
- data/lib/context_dev/resources/web.rb +26 -23
- data/lib/context_dev/version.rb +1 -1
- data/lib/context_dev.rb +3 -0
- data/rbi/context_dev/client.rbi +4 -0
- data/rbi/context_dev/models/batch_get_results_response.rbi +71 -2
- data/rbi/context_dev/models/batch_submit_params.rbi +58 -592
- data/rbi/context_dev/models/brand_search_params.rbi +71 -2
- data/rbi/context_dev/models/news_search_params.rbi +1294 -0
- data/rbi/context_dev/models/news_search_response.rbi +423 -0
- data/rbi/context_dev/models/parse_handle_params.rbi +20 -316
- data/rbi/context_dev/models/person_enrich_response.rbi +87 -0
- data/rbi/context_dev/models/utility_prefetch_params.rbi +24 -16
- data/rbi/context_dev/models/utility_prefetch_response.rbi +8 -6
- data/rbi/context_dev/models/web_screenshot_params.rbi +4 -71
- data/rbi/context_dev/models/web_web_crawl_md_response.rbi +67 -0
- data/rbi/context_dev/models/web_web_scrape_html_params.rbi +20 -356
- data/rbi/context_dev/models/web_web_scrape_html_response.rbi +65 -0
- data/rbi/context_dev/models/web_web_scrape_images_params.rbi +16 -287
- data/rbi/context_dev/models/web_web_scrape_md_params.rbi +47 -551
- data/rbi/context_dev/models/web_web_scrape_md_response.rbi +80 -0
- data/rbi/context_dev/models.rbi +2 -0
- data/rbi/context_dev/resources/brand.rbi +14 -9
- data/rbi/context_dev/resources/news.rbi +46 -0
- data/rbi/context_dev/resources/parse.rbi +5 -21
- data/rbi/context_dev/resources/utility.rbi +8 -6
- data/rbi/context_dev/resources/web.rbi +27 -66
- data/sig/context_dev/client.rbs +2 -0
- data/sig/context_dev/models/batch_get_results_response.rbs +21 -0
- data/sig/context_dev/models/batch_submit_params.rbs +54 -144
- data/sig/context_dev/models/brand_search_params.rbs +38 -1
- data/sig/context_dev/models/news_search_params.rbs +532 -0
- data/sig/context_dev/models/news_search_response.rbs +206 -0
- data/sig/context_dev/models/parse_handle_params.rbs +25 -90
- data/sig/context_dev/models/person_enrich_response.rbs +31 -0
- data/sig/context_dev/models/utility_prefetch_params.rbs +2 -1
- data/sig/context_dev/models/utility_prefetch_response.rbs +2 -1
- data/sig/context_dev/models/web_screenshot_params.rbs +5 -18
- data/sig/context_dev/models/web_web_crawl_md_response.rbs +21 -0
- data/sig/context_dev/models/web_web_scrape_html_params.rbs +24 -94
- data/sig/context_dev/models/web_web_scrape_html_response.rbs +21 -0
- data/sig/context_dev/models/web_web_scrape_images_params.rbs +20 -72
- data/sig/context_dev/models/web_web_scrape_md_params.rbs +46 -148
- data/sig/context_dev/models/web_web_scrape_md_response.rbs +28 -0
- data/sig/context_dev/models.rbs +2 -0
- data/sig/context_dev/resources/brand.rbs +3 -0
- data/sig/context_dev/resources/news.rbs +17 -0
- data/sig/context_dev/resources/parse.rbs +5 -5
- data/sig/context_dev/resources/web.rbs +12 -11
- metadata +11 -2
|
@@ -30,8 +30,7 @@ module ContextDev
|
|
|
30
30
|
optional :status, String
|
|
31
31
|
|
|
32
32
|
# @!attribute type
|
|
33
|
-
# The type of prefetch that was queued, echoed from the request
|
|
34
|
-
# 'brand')
|
|
33
|
+
# The type of prefetch that was queued, echoed from the request
|
|
35
34
|
#
|
|
36
35
|
# @return [Symbol, ContextDev::Models::UtilityPrefetchResponse::Type, nil]
|
|
37
36
|
optional :type, enum: -> { ContextDev::Models::UtilityPrefetchResponse::Type }
|
|
@@ -48,7 +47,7 @@ module ContextDev
|
|
|
48
47
|
#
|
|
49
48
|
# @param status [String] Status of the response, e.g., 'ok'
|
|
50
49
|
#
|
|
51
|
-
# @param type [Symbol, ContextDev::Models::UtilityPrefetchResponse::Type] The type of prefetch that was queued, echoed from the request
|
|
50
|
+
# @param type [Symbol, ContextDev::Models::UtilityPrefetchResponse::Type] The type of prefetch that was queued, echoed from the request
|
|
52
51
|
|
|
53
52
|
# @see ContextDev::Models::UtilityPrefetchResponse#key_metadata
|
|
54
53
|
class KeyMetadata < ContextDev::Internal::Type::BaseModel
|
|
@@ -73,14 +72,14 @@ module ContextDev
|
|
|
73
72
|
# @param credits_remaining [Integer] The number of credits remaining for your organization after this request.
|
|
74
73
|
end
|
|
75
74
|
|
|
76
|
-
# The type of prefetch that was queued, echoed from the request
|
|
77
|
-
# 'brand')
|
|
75
|
+
# The type of prefetch that was queued, echoed from the request
|
|
78
76
|
#
|
|
79
77
|
# @see ContextDev::Models::UtilityPrefetchResponse#type
|
|
80
78
|
module Type
|
|
81
79
|
extend ContextDev::Internal::Type::Enum
|
|
82
80
|
|
|
83
81
|
BRAND = :brand
|
|
82
|
+
STYLEGUIDE = :styleguide
|
|
84
83
|
|
|
85
84
|
# @!method self.values
|
|
86
85
|
# @return [Array<Symbol>]
|
|
@@ -50,8 +50,8 @@ module ContextDev
|
|
|
50
50
|
# dismiss cookie banner before capture. If 'false' or not provided, captures the
|
|
51
51
|
# page without that step.
|
|
52
52
|
#
|
|
53
|
-
# @return [Boolean,
|
|
54
|
-
optional :handle_cookie_popup,
|
|
53
|
+
# @return [Boolean, nil]
|
|
54
|
+
optional :handle_cookie_popup, ContextDev::Internal::Type::Boolean
|
|
55
55
|
|
|
56
56
|
# @!attribute max_age_ms
|
|
57
57
|
# Return a cached screenshot if a prior screenshot for the same parameters exists
|
|
@@ -135,7 +135,7 @@ module ContextDev
|
|
|
135
135
|
#
|
|
136
136
|
# @param full_screenshot [Symbol, ContextDev::Models::WebScreenshotParams::FullScreenshot] Optional parameter to determine screenshot type. If 'true', takes a full page sc
|
|
137
137
|
#
|
|
138
|
-
# @param handle_cookie_popup [Boolean
|
|
138
|
+
# @param handle_cookie_popup [Boolean] Optional parameter to control cookie/consent popup handling. If 'true', we dismi
|
|
139
139
|
#
|
|
140
140
|
# @param max_age_ms [Integer, nil] Return a cached screenshot if a prior screenshot for the same parameters exists
|
|
141
141
|
#
|
|
@@ -394,33 +394,6 @@ module ContextDev
|
|
|
394
394
|
# @return [Array<Symbol>]
|
|
395
395
|
end
|
|
396
396
|
|
|
397
|
-
# Optional parameter to control cookie/consent popup handling. If 'true', we
|
|
398
|
-
# dismiss cookie banner before capture. If 'false' or not provided, captures the
|
|
399
|
-
# page without that step.
|
|
400
|
-
module HandleCookiePopup
|
|
401
|
-
extend ContextDev::Internal::Type::Union
|
|
402
|
-
|
|
403
|
-
variant ContextDev::Internal::Type::Boolean
|
|
404
|
-
|
|
405
|
-
variant const: -> { ContextDev::Models::WebScreenshotParams::HandleCookiePopup::TRUE }
|
|
406
|
-
|
|
407
|
-
variant const: -> { ContextDev::Models::WebScreenshotParams::HandleCookiePopup::FALSE }
|
|
408
|
-
|
|
409
|
-
# @!method self.variants
|
|
410
|
-
# @return [Array(Boolean, Symbol)]
|
|
411
|
-
|
|
412
|
-
define_sorbet_constant!(:Variants) do
|
|
413
|
-
T.type_alias { T.any(T::Boolean, ContextDev::WebScreenshotParams::HandleCookiePopup::TaggedSymbol) }
|
|
414
|
-
end
|
|
415
|
-
|
|
416
|
-
# @!group
|
|
417
|
-
|
|
418
|
-
TRUE = :true
|
|
419
|
-
FALSE = :false
|
|
420
|
-
|
|
421
|
-
# @!endgroup
|
|
422
|
-
end
|
|
423
|
-
|
|
424
397
|
# Optional parameter to specify which page type to screenshot. If provided, the
|
|
425
398
|
# system will scrape the domain's links and use heuristics to find the most
|
|
426
399
|
# appropriate URL for the specified page type (30 supported languages). If not
|
|
@@ -181,6 +181,14 @@ module ContextDev
|
|
|
181
181
|
# @return [String, nil]
|
|
182
182
|
optional :favicon, String
|
|
183
183
|
|
|
184
|
+
# @!attribute headings
|
|
185
|
+
# Page headings (h1–h6) in document order, extracted from the unfiltered document.
|
|
186
|
+
# Capped at the first 500 headings. Omitted when the page has none.
|
|
187
|
+
#
|
|
188
|
+
# @return [Array<ContextDev::Models::WebWebCrawlMdResponse::Result::Metadata::Heading>, nil]
|
|
189
|
+
optional :headings,
|
|
190
|
+
-> { ContextDev::Internal::Type::ArrayOf[ContextDev::Models::WebWebCrawlMdResponse::Result::Metadata::Heading] }
|
|
191
|
+
|
|
184
192
|
# @!attribute image
|
|
185
193
|
# Primary resolved preview image from Open Graph, Twitter, or image metadata.
|
|
186
194
|
#
|
|
@@ -246,7 +254,7 @@ module ContextDev
|
|
|
246
254
|
optional :twitter,
|
|
247
255
|
-> { ContextDev::Internal::Type::HashOf[union: ContextDev::Models::WebWebCrawlMdResponse::Result::Metadata::Twitter] }
|
|
248
256
|
|
|
249
|
-
# @!method initialize(crawl_depth:, final_url:, source_url:, status_code:, success:, title:, url:, additional_meta: nil, alternates: nil, author: nil, canonical_url: nil, description: nil, favicon: nil, image: nil, json_ld: nil, keywords: nil, language: nil, modified_time: nil, open_graph: nil, published_time: nil, robots: nil, site_name: nil, twitter: nil)
|
|
257
|
+
# @!method initialize(crawl_depth:, final_url:, source_url:, status_code:, success:, title:, url:, additional_meta: nil, alternates: nil, author: nil, canonical_url: nil, description: nil, favicon: nil, headings: nil, image: nil, json_ld: nil, keywords: nil, language: nil, modified_time: nil, open_graph: nil, published_time: nil, robots: nil, site_name: nil, twitter: nil)
|
|
250
258
|
# Some parameter documentations has been truncated, see
|
|
251
259
|
# {ContextDev::Models::WebWebCrawlMdResponse::Result::Metadata} for more details.
|
|
252
260
|
#
|
|
@@ -276,6 +284,8 @@ module ContextDev
|
|
|
276
284
|
#
|
|
277
285
|
# @param favicon [String] Resolved favicon URL, when present.
|
|
278
286
|
#
|
|
287
|
+
# @param headings [Array<ContextDev::Models::WebWebCrawlMdResponse::Result::Metadata::Heading>] Page headings (h1–h6) in document order, extracted from the unfiltered document.
|
|
288
|
+
#
|
|
279
289
|
# @param image [String] Primary resolved preview image from Open Graph, Twitter, or image metadata.
|
|
280
290
|
#
|
|
281
291
|
# @param json_ld [Array<Hash{Symbol=>Object}>] JSON-LD structured data blocks parsed from the page.
|
|
@@ -345,6 +355,25 @@ module ContextDev
|
|
|
345
355
|
# @param type [String] Alternate resource MIME type, when present.
|
|
346
356
|
end
|
|
347
357
|
|
|
358
|
+
class Heading < ContextDev::Internal::Type::BaseModel
|
|
359
|
+
# @!attribute level
|
|
360
|
+
# Heading level, 1–6 (from h1–h6).
|
|
361
|
+
#
|
|
362
|
+
# @return [Integer]
|
|
363
|
+
required :level, Integer
|
|
364
|
+
|
|
365
|
+
# @!attribute text
|
|
366
|
+
# Heading text with whitespace collapsed, truncated to 1000 characters.
|
|
367
|
+
#
|
|
368
|
+
# @return [String]
|
|
369
|
+
required :text, String
|
|
370
|
+
|
|
371
|
+
# @!method initialize(level:, text:)
|
|
372
|
+
# @param level [Integer] Heading level, 1–6 (from h1–h6).
|
|
373
|
+
#
|
|
374
|
+
# @param text [String] Heading text with whitespace collapsed, truncated to 1000 characters.
|
|
375
|
+
end
|
|
376
|
+
|
|
348
377
|
module OpenGraph
|
|
349
378
|
extend ContextDev::Internal::Type::Union
|
|
350
379
|
|
|
@@ -49,8 +49,8 @@ module ContextDev
|
|
|
49
49
|
# @!attribute include_frames
|
|
50
50
|
# When true, iframes are rendered inline into the returned HTML.
|
|
51
51
|
#
|
|
52
|
-
# @return [Boolean,
|
|
53
|
-
optional :include_frames,
|
|
52
|
+
# @return [Boolean, nil]
|
|
53
|
+
optional :include_frames, ContextDev::Internal::Type::Boolean
|
|
54
54
|
|
|
55
55
|
# @!attribute include_selectors
|
|
56
56
|
# CSS selectors. When provided, only matching subtrees (and their descendants) are
|
|
@@ -80,8 +80,8 @@ module ContextDev
|
|
|
80
80
|
# extracting HTML. Defaults to false. This adds a bit of latency in exchange for
|
|
81
81
|
# more stable output on animated pages.
|
|
82
82
|
#
|
|
83
|
-
# @return [Boolean,
|
|
84
|
-
optional :settle_animations,
|
|
83
|
+
# @return [Boolean, nil]
|
|
84
|
+
optional :settle_animations, ContextDev::Internal::Type::Boolean
|
|
85
85
|
|
|
86
86
|
# @!attribute tags
|
|
87
87
|
# Optional comma-separated caller-defined tags for tracking this request. Tags are
|
|
@@ -103,8 +103,8 @@ module ContextDev
|
|
|
103
103
|
# When true, return only the page's main content in the HTML response, excluding
|
|
104
104
|
# headers, footers, sidebars, and navigation when detectable.
|
|
105
105
|
#
|
|
106
|
-
# @return [Boolean,
|
|
107
|
-
optional :use_main_content_only,
|
|
106
|
+
# @return [Boolean, nil]
|
|
107
|
+
optional :use_main_content_only, ContextDev::Internal::Type::Boolean
|
|
108
108
|
|
|
109
109
|
# @!attribute wait_for_ms
|
|
110
110
|
# Optional browser wait time in milliseconds after initial page load. Min: 0. Max:
|
|
@@ -136,7 +136,7 @@ module ContextDev
|
|
|
136
136
|
#
|
|
137
137
|
# @param headers [Hash{Symbol=>String}] Optional outbound HTTP headers forwarded only to the target URL, sent as deep-ob
|
|
138
138
|
#
|
|
139
|
-
# @param include_frames [Boolean
|
|
139
|
+
# @param include_frames [Boolean] When true, iframes are rendered inline into the returned HTML.
|
|
140
140
|
#
|
|
141
141
|
# @param include_selectors [Array<String>, nil] CSS selectors. When provided, only matching subtrees (and their descendants) are
|
|
142
142
|
#
|
|
@@ -144,13 +144,13 @@ module ContextDev
|
|
|
144
144
|
#
|
|
145
145
|
# @param pdf [ContextDev::Models::WebWebScrapeHTMLParams::Pdf] PDF parsing controls. Use start/end to limit text extraction and embedded-image
|
|
146
146
|
#
|
|
147
|
-
# @param settle_animations [Boolean
|
|
147
|
+
# @param settle_animations [Boolean] When true, waits briefly for CSS and transition animations to settle before extr
|
|
148
148
|
#
|
|
149
149
|
# @param tags [Array<String>] Optional comma-separated caller-defined tags for tracking this request. Tags are
|
|
150
150
|
#
|
|
151
151
|
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
152
152
|
#
|
|
153
|
-
# @param use_main_content_only [Boolean
|
|
153
|
+
# @param use_main_content_only [Boolean] When true, return only the page's main content in the HTML response, excluding h
|
|
154
154
|
#
|
|
155
155
|
# @param wait_for_ms [Integer, nil] Optional browser wait time in milliseconds after initial page load. Min: 0. Max:
|
|
156
156
|
#
|
|
@@ -425,31 +425,6 @@ module ContextDev
|
|
|
425
425
|
# @return [Array<Symbol>]
|
|
426
426
|
end
|
|
427
427
|
|
|
428
|
-
# When true, iframes are rendered inline into the returned HTML.
|
|
429
|
-
module IncludeFrames
|
|
430
|
-
extend ContextDev::Internal::Type::Union
|
|
431
|
-
|
|
432
|
-
variant ContextDev::Internal::Type::Boolean
|
|
433
|
-
|
|
434
|
-
variant const: -> { ContextDev::Models::WebWebScrapeHTMLParams::IncludeFrames::TRUE }
|
|
435
|
-
|
|
436
|
-
variant const: -> { ContextDev::Models::WebWebScrapeHTMLParams::IncludeFrames::FALSE }
|
|
437
|
-
|
|
438
|
-
# @!method self.variants
|
|
439
|
-
# @return [Array(Boolean, Symbol)]
|
|
440
|
-
|
|
441
|
-
define_sorbet_constant!(:Variants) do
|
|
442
|
-
T.type_alias { T.any(T::Boolean, ContextDev::WebWebScrapeHTMLParams::IncludeFrames::TaggedSymbol) }
|
|
443
|
-
end
|
|
444
|
-
|
|
445
|
-
# @!group
|
|
446
|
-
|
|
447
|
-
TRUE = :true
|
|
448
|
-
FALSE = :false
|
|
449
|
-
|
|
450
|
-
# @!endgroup
|
|
451
|
-
end
|
|
452
|
-
|
|
453
428
|
class Pdf < ContextDev::Internal::Type::BaseModel
|
|
454
429
|
# @!attribute end_
|
|
455
430
|
# Last 1-based PDF page to parse. When omitted, parsing ends at the last page.
|
|
@@ -464,17 +439,15 @@ module ContextDev
|
|
|
464
439
|
# text layer keep it. Billed at 1 credit per page OCR actually recovered, on top
|
|
465
440
|
# of the base request cost. When false, no OCR runs.
|
|
466
441
|
#
|
|
467
|
-
# @return [Boolean,
|
|
468
|
-
optional :ocr,
|
|
442
|
+
# @return [Boolean, nil]
|
|
443
|
+
optional :ocr, ContextDev::Internal::Type::Boolean
|
|
469
444
|
|
|
470
445
|
# @!attribute should_parse
|
|
471
446
|
# When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and
|
|
472
447
|
# a 400 PDF_SKIPPED is returned.
|
|
473
448
|
#
|
|
474
|
-
# @return [Boolean,
|
|
475
|
-
optional :should_parse,
|
|
476
|
-
union: -> { ContextDev::WebWebScrapeHTMLParams::Pdf::ShouldParse },
|
|
477
|
-
api_name: :shouldParse
|
|
449
|
+
# @return [Boolean, nil]
|
|
450
|
+
optional :should_parse, ContextDev::Internal::Type::Boolean, api_name: :shouldParse
|
|
478
451
|
|
|
479
452
|
# @!attribute start
|
|
480
453
|
# First 1-based PDF page to parse. When omitted, parsing starts at the first page.
|
|
@@ -491,122 +464,11 @@ module ContextDev
|
|
|
491
464
|
#
|
|
492
465
|
# @param end_ [Integer] Last 1-based PDF page to parse. When omitted, parsing ends at the last page. Mus
|
|
493
466
|
#
|
|
494
|
-
# @param ocr [Boolean
|
|
467
|
+
# @param ocr [Boolean] When true, OCR the selected PDF pages that have no usable text layer (scans), re
|
|
495
468
|
#
|
|
496
|
-
# @param should_parse [Boolean
|
|
469
|
+
# @param should_parse [Boolean] When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and
|
|
497
470
|
#
|
|
498
471
|
# @param start [Integer] First 1-based PDF page to parse. When omitted, parsing starts at the first page.
|
|
499
|
-
|
|
500
|
-
# When true, OCR the selected PDF pages that have no usable text layer (scans),
|
|
501
|
-
# replacing each recovered page's text with the OCR result while pages with a real
|
|
502
|
-
# text layer keep it. Billed at 1 credit per page OCR actually recovered, on top
|
|
503
|
-
# of the base request cost. When false, no OCR runs.
|
|
504
|
-
#
|
|
505
|
-
# @see ContextDev::Models::WebWebScrapeHTMLParams::Pdf#ocr
|
|
506
|
-
module Ocr
|
|
507
|
-
extend ContextDev::Internal::Type::Union
|
|
508
|
-
|
|
509
|
-
variant ContextDev::Internal::Type::Boolean
|
|
510
|
-
|
|
511
|
-
variant const: -> { ContextDev::Models::WebWebScrapeHTMLParams::Pdf::Ocr::TRUE }
|
|
512
|
-
|
|
513
|
-
variant const: -> { ContextDev::Models::WebWebScrapeHTMLParams::Pdf::Ocr::FALSE }
|
|
514
|
-
|
|
515
|
-
# @!method self.variants
|
|
516
|
-
# @return [Array(Boolean, Symbol)]
|
|
517
|
-
|
|
518
|
-
define_sorbet_constant!(:Variants) do
|
|
519
|
-
T.type_alias { T.any(T::Boolean, ContextDev::WebWebScrapeHTMLParams::Pdf::Ocr::TaggedSymbol) }
|
|
520
|
-
end
|
|
521
|
-
|
|
522
|
-
# @!group
|
|
523
|
-
|
|
524
|
-
TRUE = :true
|
|
525
|
-
FALSE = :false
|
|
526
|
-
|
|
527
|
-
# @!endgroup
|
|
528
|
-
end
|
|
529
|
-
|
|
530
|
-
# When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and
|
|
531
|
-
# a 400 PDF_SKIPPED is returned.
|
|
532
|
-
#
|
|
533
|
-
# @see ContextDev::Models::WebWebScrapeHTMLParams::Pdf#should_parse
|
|
534
|
-
module ShouldParse
|
|
535
|
-
extend ContextDev::Internal::Type::Union
|
|
536
|
-
|
|
537
|
-
variant ContextDev::Internal::Type::Boolean
|
|
538
|
-
|
|
539
|
-
variant const: -> { ContextDev::Models::WebWebScrapeHTMLParams::Pdf::ShouldParse::TRUE }
|
|
540
|
-
|
|
541
|
-
variant const: -> { ContextDev::Models::WebWebScrapeHTMLParams::Pdf::ShouldParse::FALSE }
|
|
542
|
-
|
|
543
|
-
# @!method self.variants
|
|
544
|
-
# @return [Array(Boolean, Symbol)]
|
|
545
|
-
|
|
546
|
-
define_sorbet_constant!(:Variants) do
|
|
547
|
-
T.type_alias { T.any(T::Boolean, ContextDev::WebWebScrapeHTMLParams::Pdf::ShouldParse::TaggedSymbol) }
|
|
548
|
-
end
|
|
549
|
-
|
|
550
|
-
# @!group
|
|
551
|
-
|
|
552
|
-
TRUE = :true
|
|
553
|
-
FALSE = :false
|
|
554
|
-
|
|
555
|
-
# @!endgroup
|
|
556
|
-
end
|
|
557
|
-
end
|
|
558
|
-
|
|
559
|
-
# When true, waits briefly for CSS and transition animations to settle before
|
|
560
|
-
# extracting HTML. Defaults to false. This adds a bit of latency in exchange for
|
|
561
|
-
# more stable output on animated pages.
|
|
562
|
-
module SettleAnimations
|
|
563
|
-
extend ContextDev::Internal::Type::Union
|
|
564
|
-
|
|
565
|
-
variant ContextDev::Internal::Type::Boolean
|
|
566
|
-
|
|
567
|
-
variant const: -> { ContextDev::Models::WebWebScrapeHTMLParams::SettleAnimations::TRUE }
|
|
568
|
-
|
|
569
|
-
variant const: -> { ContextDev::Models::WebWebScrapeHTMLParams::SettleAnimations::FALSE }
|
|
570
|
-
|
|
571
|
-
# @!method self.variants
|
|
572
|
-
# @return [Array(Boolean, Symbol)]
|
|
573
|
-
|
|
574
|
-
define_sorbet_constant!(:Variants) do
|
|
575
|
-
T.type_alias { T.any(T::Boolean, ContextDev::WebWebScrapeHTMLParams::SettleAnimations::TaggedSymbol) }
|
|
576
|
-
end
|
|
577
|
-
|
|
578
|
-
# @!group
|
|
579
|
-
|
|
580
|
-
TRUE = :true
|
|
581
|
-
FALSE = :false
|
|
582
|
-
|
|
583
|
-
# @!endgroup
|
|
584
|
-
end
|
|
585
|
-
|
|
586
|
-
# When true, return only the page's main content in the HTML response, excluding
|
|
587
|
-
# headers, footers, sidebars, and navigation when detectable.
|
|
588
|
-
module UseMainContentOnly
|
|
589
|
-
extend ContextDev::Internal::Type::Union
|
|
590
|
-
|
|
591
|
-
variant ContextDev::Internal::Type::Boolean
|
|
592
|
-
|
|
593
|
-
variant const: -> { ContextDev::Models::WebWebScrapeHTMLParams::UseMainContentOnly::TRUE }
|
|
594
|
-
|
|
595
|
-
variant const: -> { ContextDev::Models::WebWebScrapeHTMLParams::UseMainContentOnly::FALSE }
|
|
596
|
-
|
|
597
|
-
# @!method self.variants
|
|
598
|
-
# @return [Array(Boolean, Symbol)]
|
|
599
|
-
|
|
600
|
-
define_sorbet_constant!(:Variants) do
|
|
601
|
-
T.type_alias { T.any(T::Boolean, ContextDev::WebWebScrapeHTMLParams::UseMainContentOnly::TaggedSymbol) }
|
|
602
|
-
end
|
|
603
|
-
|
|
604
|
-
# @!group
|
|
605
|
-
|
|
606
|
-
TRUE = :true
|
|
607
|
-
FALSE = :false
|
|
608
|
-
|
|
609
|
-
# @!endgroup
|
|
610
472
|
end
|
|
611
473
|
|
|
612
474
|
# Set to enabled to bypass shared caches and omit request and response content
|
|
@@ -135,6 +135,14 @@ module ContextDev
|
|
|
135
135
|
# @return [String, nil]
|
|
136
136
|
optional :favicon, String
|
|
137
137
|
|
|
138
|
+
# @!attribute headings
|
|
139
|
+
# Page headings (h1–h6) in document order, extracted from the unfiltered document.
|
|
140
|
+
# Capped at the first 500 headings. Omitted when the page has none.
|
|
141
|
+
#
|
|
142
|
+
# @return [Array<ContextDev::Models::WebWebScrapeHTMLResponse::Metadata::Heading>, nil]
|
|
143
|
+
optional :headings,
|
|
144
|
+
-> { ContextDev::Internal::Type::ArrayOf[ContextDev::Models::WebWebScrapeHTMLResponse::Metadata::Heading] }
|
|
145
|
+
|
|
138
146
|
# @!attribute image
|
|
139
147
|
# Primary resolved preview image from Open Graph, Twitter, or image metadata.
|
|
140
148
|
#
|
|
@@ -206,7 +214,7 @@ module ContextDev
|
|
|
206
214
|
optional :twitter,
|
|
207
215
|
-> { ContextDev::Internal::Type::HashOf[union: ContextDev::Models::WebWebScrapeHTMLResponse::Metadata::Twitter] }
|
|
208
216
|
|
|
209
|
-
# @!method initialize(final_url:, source_url:, additional_meta: nil, alternates: nil, author: nil, canonical_url: nil, description: nil, favicon: nil, image: nil, json_ld: nil, keywords: nil, language: nil, modified_time: nil, open_graph: nil, published_time: nil, robots: nil, site_name: nil, title: nil, twitter: nil)
|
|
217
|
+
# @!method initialize(final_url:, source_url:, additional_meta: nil, alternates: nil, author: nil, canonical_url: nil, description: nil, favicon: nil, headings: nil, image: nil, json_ld: nil, keywords: nil, language: nil, modified_time: nil, open_graph: nil, published_time: nil, robots: nil, site_name: nil, title: nil, twitter: nil)
|
|
210
218
|
# Some parameter documentations has been truncated, see
|
|
211
219
|
# {ContextDev::Models::WebWebScrapeHTMLResponse::Metadata} for more details.
|
|
212
220
|
#
|
|
@@ -228,6 +236,8 @@ module ContextDev
|
|
|
228
236
|
#
|
|
229
237
|
# @param favicon [String] Resolved favicon URL, when present.
|
|
230
238
|
#
|
|
239
|
+
# @param headings [Array<ContextDev::Models::WebWebScrapeHTMLResponse::Metadata::Heading>] Page headings (h1–h6) in document order, extracted from the unfiltered document.
|
|
240
|
+
#
|
|
231
241
|
# @param image [String] Primary resolved preview image from Open Graph, Twitter, or image metadata.
|
|
232
242
|
#
|
|
233
243
|
# @param json_ld [Array<Hash{Symbol=>Object}>] JSON-LD structured data blocks parsed from the page.
|
|
@@ -299,6 +309,25 @@ module ContextDev
|
|
|
299
309
|
# @param type [String] Alternate resource MIME type, when present.
|
|
300
310
|
end
|
|
301
311
|
|
|
312
|
+
class Heading < ContextDev::Internal::Type::BaseModel
|
|
313
|
+
# @!attribute level
|
|
314
|
+
# Heading level, 1–6 (from h1–h6).
|
|
315
|
+
#
|
|
316
|
+
# @return [Integer]
|
|
317
|
+
required :level, Integer
|
|
318
|
+
|
|
319
|
+
# @!attribute text
|
|
320
|
+
# Heading text with whitespace collapsed, truncated to 1000 characters.
|
|
321
|
+
#
|
|
322
|
+
# @return [String]
|
|
323
|
+
required :text, String
|
|
324
|
+
|
|
325
|
+
# @!method initialize(level:, text:)
|
|
326
|
+
# @param level [Integer] Heading level, 1–6 (from h1–h6).
|
|
327
|
+
#
|
|
328
|
+
# @param text [String] Heading text with whitespace collapsed, truncated to 1000 characters.
|
|
329
|
+
end
|
|
330
|
+
|
|
302
331
|
module OpenGraph
|
|
303
332
|
extend ContextDev::Internal::Type::Union
|
|
304
333
|
|
|
@@ -31,8 +31,8 @@ module ContextDev
|
|
|
31
31
|
# group is kept. Images that cannot be downloaded or hashed are kept. Default:
|
|
32
32
|
# false.
|
|
33
33
|
#
|
|
34
|
-
# @return [Boolean,
|
|
35
|
-
optional :dedupe,
|
|
34
|
+
# @return [Boolean, nil]
|
|
35
|
+
optional :dedupe, ContextDev::Internal::Type::Boolean
|
|
36
36
|
|
|
37
37
|
# @!attribute enrichment
|
|
38
38
|
# Optional per-image processing, sent as deep-object query params such as
|
|
@@ -87,7 +87,7 @@ module ContextDev
|
|
|
87
87
|
#
|
|
88
88
|
# @param actions [Array<ContextDev::Models::WebWebScrapeImagesParams::Action::Wait, ContextDev::Models::WebWebScrapeImagesParams::Action::Perform>, nil] Optional browser actions executed in array order after the page loads and before
|
|
89
89
|
#
|
|
90
|
-
# @param dedupe [Boolean
|
|
90
|
+
# @param dedupe [Boolean] When true, visually duplicate images are removed: every image is loaded and perc
|
|
91
91
|
#
|
|
92
92
|
# @param enrichment [ContextDev::Models::WebWebScrapeImagesParams::Enrichment, nil] Optional per-image processing, sent as deep-object query params such as enrichme
|
|
93
93
|
#
|
|
@@ -156,49 +156,19 @@ module ContextDev
|
|
|
156
156
|
# @return [Array(ContextDev::Models::WebWebScrapeImagesParams::Action::Wait, ContextDev::Models::WebWebScrapeImagesParams::Action::Perform)]
|
|
157
157
|
end
|
|
158
158
|
|
|
159
|
-
# When true, visually duplicate images are removed: every image is loaded and
|
|
160
|
-
# perceptually hashed, and only the highest-resolution copy of each duplicate
|
|
161
|
-
# group is kept. Images that cannot be downloaded or hashed are kept. Default:
|
|
162
|
-
# false.
|
|
163
|
-
module Dedupe
|
|
164
|
-
extend ContextDev::Internal::Type::Union
|
|
165
|
-
|
|
166
|
-
variant ContextDev::Internal::Type::Boolean
|
|
167
|
-
|
|
168
|
-
variant const: -> { ContextDev::Models::WebWebScrapeImagesParams::Dedupe::TRUE }
|
|
169
|
-
|
|
170
|
-
variant const: -> { ContextDev::Models::WebWebScrapeImagesParams::Dedupe::FALSE }
|
|
171
|
-
|
|
172
|
-
# @!method self.variants
|
|
173
|
-
# @return [Array(Boolean, Symbol)]
|
|
174
|
-
|
|
175
|
-
define_sorbet_constant!(:Variants) do
|
|
176
|
-
T.type_alias { T.any(T::Boolean, ContextDev::WebWebScrapeImagesParams::Dedupe::TaggedSymbol) }
|
|
177
|
-
end
|
|
178
|
-
|
|
179
|
-
# @!group
|
|
180
|
-
|
|
181
|
-
TRUE = :true
|
|
182
|
-
FALSE = :false
|
|
183
|
-
|
|
184
|
-
# @!endgroup
|
|
185
|
-
end
|
|
186
|
-
|
|
187
159
|
class Enrichment < ContextDev::Internal::Type::BaseModel
|
|
188
160
|
# @!attribute classification
|
|
189
161
|
# Classify each image by visual asset type.
|
|
190
162
|
#
|
|
191
|
-
# @return [Boolean,
|
|
192
|
-
optional :classification,
|
|
163
|
+
# @return [Boolean, nil]
|
|
164
|
+
optional :classification, ContextDev::Internal::Type::Boolean
|
|
193
165
|
|
|
194
166
|
# @!attribute hosted_url
|
|
195
167
|
# Host materializable images on the Brand.dev CDN and return their URL and MIME
|
|
196
168
|
# type.
|
|
197
169
|
#
|
|
198
|
-
# @return [Boolean,
|
|
199
|
-
optional :hosted_url,
|
|
200
|
-
union: -> { ContextDev::WebWebScrapeImagesParams::Enrichment::HostedURL },
|
|
201
|
-
api_name: :hostedUrl
|
|
170
|
+
# @return [Boolean, nil]
|
|
171
|
+
optional :hosted_url, ContextDev::Internal::Type::Boolean, api_name: :hostedUrl
|
|
202
172
|
|
|
203
173
|
# @!attribute max_time_per_ms
|
|
204
174
|
# Per-image enrichment timeout in milliseconds. Default: 30000. Maximum: 60000.
|
|
@@ -209,8 +179,8 @@ module ContextDev
|
|
|
209
179
|
# @!attribute resolution
|
|
210
180
|
# Measure image width and height when possible.
|
|
211
181
|
#
|
|
212
|
-
# @return [Boolean,
|
|
213
|
-
optional :resolution,
|
|
182
|
+
# @return [Boolean, nil]
|
|
183
|
+
optional :resolution, ContextDev::Internal::Type::Boolean
|
|
214
184
|
|
|
215
185
|
# @!method initialize(classification: nil, hosted_url: nil, max_time_per_ms: nil, resolution: nil)
|
|
216
186
|
# Some parameter documentations has been truncated, see
|
|
@@ -219,95 +189,13 @@ module ContextDev
|
|
|
219
189
|
# Optional per-image processing, sent as deep-object query params such as
|
|
220
190
|
# enrichment[resolution]=true.
|
|
221
191
|
#
|
|
222
|
-
# @param classification [Boolean
|
|
192
|
+
# @param classification [Boolean] Classify each image by visual asset type.
|
|
223
193
|
#
|
|
224
|
-
# @param hosted_url [Boolean
|
|
194
|
+
# @param hosted_url [Boolean] Host materializable images on the Brand.dev CDN and return their URL and MIME ty
|
|
225
195
|
#
|
|
226
196
|
# @param max_time_per_ms [Integer] Per-image enrichment timeout in milliseconds. Default: 30000. Maximum: 60000.
|
|
227
197
|
#
|
|
228
|
-
# @param resolution [Boolean
|
|
229
|
-
|
|
230
|
-
# Classify each image by visual asset type.
|
|
231
|
-
#
|
|
232
|
-
# @see ContextDev::Models::WebWebScrapeImagesParams::Enrichment#classification
|
|
233
|
-
module Classification
|
|
234
|
-
extend ContextDev::Internal::Type::Union
|
|
235
|
-
|
|
236
|
-
variant ContextDev::Internal::Type::Boolean
|
|
237
|
-
|
|
238
|
-
variant const: -> { ContextDev::Models::WebWebScrapeImagesParams::Enrichment::Classification::TRUE }
|
|
239
|
-
|
|
240
|
-
variant const: -> { ContextDev::Models::WebWebScrapeImagesParams::Enrichment::Classification::FALSE }
|
|
241
|
-
|
|
242
|
-
# @!method self.variants
|
|
243
|
-
# @return [Array(Boolean, Symbol)]
|
|
244
|
-
|
|
245
|
-
define_sorbet_constant!(:Variants) do
|
|
246
|
-
T.type_alias { T.any(T::Boolean, ContextDev::WebWebScrapeImagesParams::Enrichment::Classification::TaggedSymbol) }
|
|
247
|
-
end
|
|
248
|
-
|
|
249
|
-
# @!group
|
|
250
|
-
|
|
251
|
-
TRUE = :true
|
|
252
|
-
FALSE = :false
|
|
253
|
-
|
|
254
|
-
# @!endgroup
|
|
255
|
-
end
|
|
256
|
-
|
|
257
|
-
# Host materializable images on the Brand.dev CDN and return their URL and MIME
|
|
258
|
-
# type.
|
|
259
|
-
#
|
|
260
|
-
# @see ContextDev::Models::WebWebScrapeImagesParams::Enrichment#hosted_url
|
|
261
|
-
module HostedURL
|
|
262
|
-
extend ContextDev::Internal::Type::Union
|
|
263
|
-
|
|
264
|
-
variant ContextDev::Internal::Type::Boolean
|
|
265
|
-
|
|
266
|
-
variant const: -> { ContextDev::Models::WebWebScrapeImagesParams::Enrichment::HostedURL::TRUE }
|
|
267
|
-
|
|
268
|
-
variant const: -> { ContextDev::Models::WebWebScrapeImagesParams::Enrichment::HostedURL::FALSE }
|
|
269
|
-
|
|
270
|
-
# @!method self.variants
|
|
271
|
-
# @return [Array(Boolean, Symbol)]
|
|
272
|
-
|
|
273
|
-
define_sorbet_constant!(:Variants) do
|
|
274
|
-
T.type_alias { T.any(T::Boolean, ContextDev::WebWebScrapeImagesParams::Enrichment::HostedURL::TaggedSymbol) }
|
|
275
|
-
end
|
|
276
|
-
|
|
277
|
-
# @!group
|
|
278
|
-
|
|
279
|
-
TRUE = :true
|
|
280
|
-
FALSE = :false
|
|
281
|
-
|
|
282
|
-
# @!endgroup
|
|
283
|
-
end
|
|
284
|
-
|
|
285
|
-
# Measure image width and height when possible.
|
|
286
|
-
#
|
|
287
|
-
# @see ContextDev::Models::WebWebScrapeImagesParams::Enrichment#resolution
|
|
288
|
-
module Resolution
|
|
289
|
-
extend ContextDev::Internal::Type::Union
|
|
290
|
-
|
|
291
|
-
variant ContextDev::Internal::Type::Boolean
|
|
292
|
-
|
|
293
|
-
variant const: -> { ContextDev::Models::WebWebScrapeImagesParams::Enrichment::Resolution::TRUE }
|
|
294
|
-
|
|
295
|
-
variant const: -> { ContextDev::Models::WebWebScrapeImagesParams::Enrichment::Resolution::FALSE }
|
|
296
|
-
|
|
297
|
-
# @!method self.variants
|
|
298
|
-
# @return [Array(Boolean, Symbol)]
|
|
299
|
-
|
|
300
|
-
define_sorbet_constant!(:Variants) do
|
|
301
|
-
T.type_alias { T.any(T::Boolean, ContextDev::WebWebScrapeImagesParams::Enrichment::Resolution::TaggedSymbol) }
|
|
302
|
-
end
|
|
303
|
-
|
|
304
|
-
# @!group
|
|
305
|
-
|
|
306
|
-
TRUE = :true
|
|
307
|
-
FALSE = :false
|
|
308
|
-
|
|
309
|
-
# @!endgroup
|
|
310
|
-
end
|
|
198
|
+
# @param resolution [Boolean] Measure image width and height when possible.
|
|
311
199
|
end
|
|
312
200
|
end
|
|
313
201
|
end
|