context.dev 2.6.0 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +18 -0
- data/README.md +1 -1
- data/lib/context_dev/models/batch_cancel_response.rb +70 -183
- data/lib/context_dev/models/batch_list_response.rb +86 -95
- data/lib/context_dev/models/batch_retrieve_response.rb +86 -103
- data/lib/context_dev/models/brand_search_params.rb +36 -0
- data/lib/context_dev/models/brand_search_response.rb +84 -0
- data/lib/context_dev/models/crawl_controls.rb +126 -0
- data/lib/context_dev/models/failure.rb +27 -0
- data/lib/context_dev/models/intake.rb +62 -0
- data/lib/context_dev/models/{error_count.rb → page_error_count.rb} +1 -1
- data/lib/context_dev/models/web_screenshot_params.rb +5 -7
- data/lib/context_dev/models/web_web_crawl_md_params.rb +5 -7
- data/lib/context_dev/models/web_web_scrape_html_params.rb +5 -7
- data/lib/context_dev/models/web_web_scrape_md_params.rb +5 -7
- data/lib/context_dev/models.rb +8 -2
- data/lib/context_dev/resources/batch.rb +3 -6
- data/lib/context_dev/resources/brand.rb +34 -0
- data/lib/context_dev/resources/web.rb +4 -4
- data/lib/context_dev/version.rb +1 -1
- data/lib/context_dev.rb +6 -2
- data/rbi/context_dev/models/batch_cancel_response.rbi +98 -316
- data/rbi/context_dev/models/batch_list_response.rbi +127 -152
- data/rbi/context_dev/models/batch_retrieve_response.rbi +126 -161
- data/rbi/context_dev/models/brand_search_params.rbi +60 -0
- data/rbi/context_dev/models/brand_search_response.rbi +146 -0
- data/rbi/context_dev/models/crawl_controls.rbi +252 -0
- data/rbi/context_dev/models/{error.rbi → failure.rbi} +10 -7
- data/rbi/context_dev/models/intake.rbi +88 -0
- data/rbi/context_dev/models/{error_count.rbi → page_error_count.rbi} +2 -2
- data/rbi/context_dev/models/web_screenshot_params.rbi +6 -9
- data/rbi/context_dev/models/web_web_crawl_md_params.rbi +6 -9
- data/rbi/context_dev/models/web_web_scrape_html_params.rbi +6 -9
- data/rbi/context_dev/models/web_web_scrape_md_params.rbi +6 -9
- data/rbi/context_dev/models.rbi +8 -2
- data/rbi/context_dev/resources/batch.rbi +3 -6
- data/rbi/context_dev/resources/brand.rbi +26 -0
- data/rbi/context_dev/resources/web.rbi +8 -12
- data/sig/context_dev/models/batch_cancel_response.rbs +32 -124
- data/sig/context_dev/models/batch_list_response.rbs +42 -63
- data/sig/context_dev/models/batch_retrieve_response.rbs +42 -70
- data/sig/context_dev/models/brand_search_params.rbs +30 -0
- data/sig/context_dev/models/brand_search_response.rbs +62 -0
- data/sig/context_dev/models/crawl_controls.rbs +114 -0
- data/sig/context_dev/models/{error.rbs → failure.rbs} +2 -2
- data/sig/context_dev/models/intake.rbs +40 -0
- data/sig/context_dev/models/{error_count.rbs → page_error_count.rbs} +2 -2
- data/sig/context_dev/models.rbs +8 -2
- data/sig/context_dev/resources/brand.rbs +6 -0
- metadata +20 -8
- data/lib/context_dev/models/error.rb +0 -26
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ContextDev
|
|
4
|
+
module Models
|
|
5
|
+
# @see ContextDev::Resources::Brand#search
|
|
6
|
+
class BrandSearchResponse < ContextDev::Internal::Type::BaseModel
|
|
7
|
+
# @!attribute results
|
|
8
|
+
# Up to 10 matching brands, most popular first. Empty when nothing matches.
|
|
9
|
+
#
|
|
10
|
+
# @return [Array<ContextDev::Models::BrandSearchResponse::Result>]
|
|
11
|
+
required :results,
|
|
12
|
+
-> { ContextDev::Internal::Type::ArrayOf[ContextDev::Models::BrandSearchResponse::Result] }
|
|
13
|
+
|
|
14
|
+
# @!attribute key_metadata
|
|
15
|
+
# Metadata about the API key used for the request. Included in every response
|
|
16
|
+
# whenever a valid API key is provided, even when the response status is not 200.
|
|
17
|
+
#
|
|
18
|
+
# @return [ContextDev::Models::BrandSearchResponse::KeyMetadata, nil]
|
|
19
|
+
optional :key_metadata, -> { ContextDev::Models::BrandSearchResponse::KeyMetadata }
|
|
20
|
+
|
|
21
|
+
# @!method initialize(results:, key_metadata: nil)
|
|
22
|
+
# Some parameter documentations has been truncated, see
|
|
23
|
+
# {ContextDev::Models::BrandSearchResponse} for more details.
|
|
24
|
+
#
|
|
25
|
+
# @param results [Array<ContextDev::Models::BrandSearchResponse::Result>] Up to 10 matching brands, most popular first. Empty when nothing matches.
|
|
26
|
+
#
|
|
27
|
+
# @param key_metadata [ContextDev::Models::BrandSearchResponse::KeyMetadata] Metadata about the API key used for the request. Included in every response when
|
|
28
|
+
|
|
29
|
+
class Result < ContextDev::Internal::Type::BaseModel
|
|
30
|
+
# @!attribute domain
|
|
31
|
+
# The brand's domain.
|
|
32
|
+
#
|
|
33
|
+
# @return [String]
|
|
34
|
+
required :domain, String
|
|
35
|
+
|
|
36
|
+
# @!attribute logo
|
|
37
|
+
# Logo link URL that serves the brand's logo, generated per request for the
|
|
38
|
+
# calling organization.
|
|
39
|
+
#
|
|
40
|
+
# @return [String]
|
|
41
|
+
required :logo, String
|
|
42
|
+
|
|
43
|
+
# @!attribute name
|
|
44
|
+
# The brand's name. Empty string when unknown.
|
|
45
|
+
#
|
|
46
|
+
# @return [String]
|
|
47
|
+
required :name, String
|
|
48
|
+
|
|
49
|
+
# @!method initialize(domain:, logo:, name:)
|
|
50
|
+
# Some parameter documentations has been truncated, see
|
|
51
|
+
# {ContextDev::Models::BrandSearchResponse::Result} for more details.
|
|
52
|
+
#
|
|
53
|
+
# @param domain [String] The brand's domain.
|
|
54
|
+
#
|
|
55
|
+
# @param logo [String] Logo link URL that serves the brand's logo, generated per request for the callin
|
|
56
|
+
#
|
|
57
|
+
# @param name [String] The brand's name. Empty string when unknown.
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# @see ContextDev::Models::BrandSearchResponse#key_metadata
|
|
61
|
+
class KeyMetadata < ContextDev::Internal::Type::BaseModel
|
|
62
|
+
# @!attribute credits_consumed
|
|
63
|
+
# The number of credits consumed by this request.
|
|
64
|
+
#
|
|
65
|
+
# @return [Integer]
|
|
66
|
+
required :credits_consumed, Integer
|
|
67
|
+
|
|
68
|
+
# @!attribute credits_remaining
|
|
69
|
+
# The number of credits remaining for your organization after this request.
|
|
70
|
+
#
|
|
71
|
+
# @return [Integer]
|
|
72
|
+
required :credits_remaining, Integer
|
|
73
|
+
|
|
74
|
+
# @!method initialize(credits_consumed:, credits_remaining:)
|
|
75
|
+
# Metadata about the API key used for the request. Included in every response
|
|
76
|
+
# whenever a valid API key is provided, even when the response status is not 200.
|
|
77
|
+
#
|
|
78
|
+
# @param credits_consumed [Integer] The number of credits consumed by this request.
|
|
79
|
+
#
|
|
80
|
+
# @param credits_remaining [Integer] The number of credits remaining for your organization after this request.
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ContextDev
|
|
4
|
+
module Models
|
|
5
|
+
class CrawlControls < ContextDev::Internal::Type::BaseModel
|
|
6
|
+
# @!attribute follow_subdomains
|
|
7
|
+
# Whether links to subdomains were followed. Always false for a sitemap crawl.
|
|
8
|
+
#
|
|
9
|
+
# @return [Boolean]
|
|
10
|
+
required :follow_subdomains, ContextDev::Internal::Type::Boolean
|
|
11
|
+
|
|
12
|
+
# @!attribute max_depth
|
|
13
|
+
# Link depth limit. Always 0 for a sitemap crawl, which never follows links off
|
|
14
|
+
# its URLs; null when a `start_url` crawl set no limit.
|
|
15
|
+
#
|
|
16
|
+
# @return [Integer, nil]
|
|
17
|
+
required :max_depth, Integer, nil?: true
|
|
18
|
+
|
|
19
|
+
# @!attribute max_pages
|
|
20
|
+
# The `maxUrls` submitted with the crawl. A sitemap crawl scrapes only the URLs
|
|
21
|
+
# its sitemap actually lists, up to this many, so `input.reserved` is often lower.
|
|
22
|
+
#
|
|
23
|
+
# @return [Integer]
|
|
24
|
+
required :max_pages, Integer
|
|
25
|
+
|
|
26
|
+
# @!attribute source
|
|
27
|
+
# Where the crawl started.
|
|
28
|
+
#
|
|
29
|
+
# @return [ContextDev::Models::CrawlControls::Source::UnionMember0, ContextDev::Models::CrawlControls::Source::UnionMember1]
|
|
30
|
+
required :source, union: -> { ContextDev::CrawlControls::Source }
|
|
31
|
+
|
|
32
|
+
# @!attribute url_pattern
|
|
33
|
+
# RE2 pattern URLs had to match to be crawled. Null when the crawl set none.
|
|
34
|
+
#
|
|
35
|
+
# @return [String, nil]
|
|
36
|
+
required :url_pattern, String, nil?: true
|
|
37
|
+
|
|
38
|
+
# @!method initialize(follow_subdomains:, max_depth:, max_pages:, source:, url_pattern:)
|
|
39
|
+
# Some parameter documentations has been truncated, see
|
|
40
|
+
# {ContextDev::Models::CrawlControls} for more details.
|
|
41
|
+
#
|
|
42
|
+
# The crawl controls as submitted, so the limits requested can be compared against
|
|
43
|
+
# what the crawl reached.
|
|
44
|
+
#
|
|
45
|
+
# @param follow_subdomains [Boolean] Whether links to subdomains were followed. Always false for a sitemap crawl.
|
|
46
|
+
#
|
|
47
|
+
# @param max_depth [Integer, nil] Link depth limit. Always 0 for a sitemap crawl, which never follows links off it
|
|
48
|
+
#
|
|
49
|
+
# @param max_pages [Integer] The `maxUrls` submitted with the crawl. A sitemap crawl scrapes only the URLs it
|
|
50
|
+
#
|
|
51
|
+
# @param source [ContextDev::Models::CrawlControls::Source::UnionMember0, ContextDev::Models::CrawlControls::Source::UnionMember1] Where the crawl started.
|
|
52
|
+
#
|
|
53
|
+
# @param url_pattern [String, nil] RE2 pattern URLs had to match to be crawled. Null when the crawl set none.
|
|
54
|
+
|
|
55
|
+
# Where the crawl started.
|
|
56
|
+
#
|
|
57
|
+
# @see ContextDev::Models::CrawlControls#source
|
|
58
|
+
module Source
|
|
59
|
+
extend ContextDev::Internal::Type::Union
|
|
60
|
+
|
|
61
|
+
variant -> { ContextDev::CrawlControls::Source::UnionMember0 }
|
|
62
|
+
|
|
63
|
+
variant -> { ContextDev::CrawlControls::Source::UnionMember1 }
|
|
64
|
+
|
|
65
|
+
class UnionMember0 < ContextDev::Internal::Type::BaseModel
|
|
66
|
+
# @!attribute type
|
|
67
|
+
#
|
|
68
|
+
# @return [Symbol, ContextDev::Models::CrawlControls::Source::UnionMember0::Type]
|
|
69
|
+
required :type, enum: -> { ContextDev::CrawlControls::Source::UnionMember0::Type }
|
|
70
|
+
|
|
71
|
+
# @!attribute url
|
|
72
|
+
# Page the crawl started from.
|
|
73
|
+
#
|
|
74
|
+
# @return [String]
|
|
75
|
+
required :url, String
|
|
76
|
+
|
|
77
|
+
# @!method initialize(type:, url:)
|
|
78
|
+
# @param type [Symbol, ContextDev::Models::CrawlControls::Source::UnionMember0::Type]
|
|
79
|
+
#
|
|
80
|
+
# @param url [String] Page the crawl started from.
|
|
81
|
+
|
|
82
|
+
# @see ContextDev::Models::CrawlControls::Source::UnionMember0#type
|
|
83
|
+
module Type
|
|
84
|
+
extend ContextDev::Internal::Type::Enum
|
|
85
|
+
|
|
86
|
+
START_URL = :start_url
|
|
87
|
+
|
|
88
|
+
# @!method self.values
|
|
89
|
+
# @return [Array<Symbol>]
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
class UnionMember1 < ContextDev::Internal::Type::BaseModel
|
|
94
|
+
# @!attribute domain
|
|
95
|
+
# Domain whose sitemap supplied the pages.
|
|
96
|
+
#
|
|
97
|
+
# @return [String]
|
|
98
|
+
required :domain, String
|
|
99
|
+
|
|
100
|
+
# @!attribute type
|
|
101
|
+
#
|
|
102
|
+
# @return [Symbol, ContextDev::Models::CrawlControls::Source::UnionMember1::Type]
|
|
103
|
+
required :type, enum: -> { ContextDev::CrawlControls::Source::UnionMember1::Type }
|
|
104
|
+
|
|
105
|
+
# @!method initialize(domain:, type:)
|
|
106
|
+
# @param domain [String] Domain whose sitemap supplied the pages.
|
|
107
|
+
#
|
|
108
|
+
# @param type [Symbol, ContextDev::Models::CrawlControls::Source::UnionMember1::Type]
|
|
109
|
+
|
|
110
|
+
# @see ContextDev::Models::CrawlControls::Source::UnionMember1#type
|
|
111
|
+
module Type
|
|
112
|
+
extend ContextDev::Internal::Type::Enum
|
|
113
|
+
|
|
114
|
+
SITEMAP = :sitemap
|
|
115
|
+
|
|
116
|
+
# @!method self.values
|
|
117
|
+
# @return [Array<Symbol>]
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# @!method self.variants
|
|
122
|
+
# @return [Array(ContextDev::Models::CrawlControls::Source::UnionMember0, ContextDev::Models::CrawlControls::Source::UnionMember1)]
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ContextDev
|
|
4
|
+
module Models
|
|
5
|
+
class Failure < ContextDev::Internal::Type::BaseModel
|
|
6
|
+
# @!attribute code
|
|
7
|
+
# Why the batch itself stopped.
|
|
8
|
+
#
|
|
9
|
+
# @return [String]
|
|
10
|
+
required :code, String
|
|
11
|
+
|
|
12
|
+
# @!attribute message
|
|
13
|
+
# Human-readable explanation.
|
|
14
|
+
#
|
|
15
|
+
# @return [String]
|
|
16
|
+
required :message, String
|
|
17
|
+
|
|
18
|
+
# @!method initialize(code:, message:)
|
|
19
|
+
# A failure of the batch as a whole, distinct from the per-page failures in
|
|
20
|
+
# `page_errors`.
|
|
21
|
+
#
|
|
22
|
+
# @param code [String] Why the batch itself stopped.
|
|
23
|
+
#
|
|
24
|
+
# @param message [String] Human-readable explanation.
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ContextDev
|
|
4
|
+
module Models
|
|
5
|
+
class Intake < ContextDev::Internal::Type::BaseModel
|
|
6
|
+
# @!attribute duplicates
|
|
7
|
+
# URLs dropped before reserving because another entry resolved to the same page.
|
|
8
|
+
# Non-zero for sitemap crawls too, whose sitemaps routinely list a page more than
|
|
9
|
+
# once.
|
|
10
|
+
#
|
|
11
|
+
# @return [Integer]
|
|
12
|
+
required :duplicates, Integer
|
|
13
|
+
|
|
14
|
+
# @!attribute invalid
|
|
15
|
+
# URLs from your list rejected as unusable; the same ones are itemised in
|
|
16
|
+
# `invalid_urls` at submission. Null for a crawl — a crawl that resolves no usable
|
|
17
|
+
# page is rejected outright with a 400 rather than accepted with an empty list.
|
|
18
|
+
#
|
|
19
|
+
# @return [Integer, nil]
|
|
20
|
+
required :invalid, Integer, nil?: true
|
|
21
|
+
|
|
22
|
+
# @!attribute reserved
|
|
23
|
+
# Pages credits were reserved for. Everything else — progress, the refund, the
|
|
24
|
+
# completion percentage — is measured against this.
|
|
25
|
+
#
|
|
26
|
+
# @return [Integer]
|
|
27
|
+
required :reserved, Integer
|
|
28
|
+
|
|
29
|
+
# @!attribute reserved_is_ceiling
|
|
30
|
+
# Whether `reserved` is an upper bound the batch may finish under. True only for a
|
|
31
|
+
# crawl that follows links, whose reachable page count is unknowable until it
|
|
32
|
+
# runs. False for a scrape and for a sitemap crawl, where `reserved` is an exact
|
|
33
|
+
# page count.
|
|
34
|
+
#
|
|
35
|
+
# @return [Boolean]
|
|
36
|
+
required :reserved_is_ceiling, ContextDev::Internal::Type::Boolean
|
|
37
|
+
|
|
38
|
+
# @!attribute submitted
|
|
39
|
+
# URLs in the list you sent, before validation and de-duplication. Null for a
|
|
40
|
+
# crawl, which is given a source rather than a list.
|
|
41
|
+
#
|
|
42
|
+
# @return [Integer, nil]
|
|
43
|
+
required :submitted, Integer, nil?: true
|
|
44
|
+
|
|
45
|
+
# @!method initialize(duplicates:, invalid:, reserved:, reserved_is_ceiling:, submitted:)
|
|
46
|
+
# Some parameter documentations has been truncated, see
|
|
47
|
+
# {ContextDev::Models::Intake} for more details.
|
|
48
|
+
#
|
|
49
|
+
# What submission took in, and what it charged for.
|
|
50
|
+
#
|
|
51
|
+
# @param duplicates [Integer] URLs dropped before reserving because another entry resolved to the same page. N
|
|
52
|
+
#
|
|
53
|
+
# @param invalid [Integer, nil] URLs from your list rejected as unusable; the same ones are itemised in `invalid
|
|
54
|
+
#
|
|
55
|
+
# @param reserved [Integer] Pages credits were reserved for. Everything else — progress, the refund, the com
|
|
56
|
+
#
|
|
57
|
+
# @param reserved_is_ceiling [Boolean] Whether `reserved` is an upper bound the batch may finish under. True only for a
|
|
58
|
+
#
|
|
59
|
+
# @param submitted [Integer, nil] URLs in the list you sent, before validation and de-duplication. Null for a craw
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
@@ -15,9 +15,8 @@ module ContextDev
|
|
|
15
15
|
optional :color_scheme, enum: -> { ContextDev::WebScreenshotParams::ColorScheme }
|
|
16
16
|
|
|
17
17
|
# @!attribute country
|
|
18
|
-
#
|
|
19
|
-
#
|
|
20
|
-
# countries. When provided, Context.dev fetches the target page from that country.
|
|
18
|
+
# Fetch the target page through a residential proxy in this country (ISO 3166-1
|
|
19
|
+
# alpha-2).
|
|
21
20
|
#
|
|
22
21
|
# @return [Symbol, ContextDev::Models::WebScreenshotParams::Country, nil]
|
|
23
22
|
optional :country, enum: -> { ContextDev::WebScreenshotParams::Country }
|
|
@@ -128,7 +127,7 @@ module ContextDev
|
|
|
128
127
|
#
|
|
129
128
|
# @param color_scheme [Symbol, ContextDev::Models::WebScreenshotParams::ColorScheme] Optional parameter to choose the site's visual theme in the screenshot. Use 'lig
|
|
130
129
|
#
|
|
131
|
-
# @param country [Symbol, ContextDev::Models::WebScreenshotParams::Country]
|
|
130
|
+
# @param country [Symbol, ContextDev::Models::WebScreenshotParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al
|
|
132
131
|
#
|
|
133
132
|
# @param direct_url [String] A specific URL to screenshot directly, bypassing domain resolution (e.g., 'https
|
|
134
133
|
#
|
|
@@ -168,9 +167,8 @@ module ContextDev
|
|
|
168
167
|
# @return [Array<Symbol>]
|
|
169
168
|
end
|
|
170
169
|
|
|
171
|
-
#
|
|
172
|
-
#
|
|
173
|
-
# countries. When provided, Context.dev fetches the target page from that country.
|
|
170
|
+
# Fetch the target page through a residential proxy in this country (ISO 3166-1
|
|
171
|
+
# alpha-2).
|
|
174
172
|
module Country
|
|
175
173
|
extend ContextDev::Internal::Type::Enum
|
|
176
174
|
|
|
@@ -14,9 +14,8 @@ module ContextDev
|
|
|
14
14
|
required :url, String
|
|
15
15
|
|
|
16
16
|
# @!attribute country
|
|
17
|
-
#
|
|
18
|
-
#
|
|
19
|
-
# countries. When provided, Context.dev fetches the target page from that country.
|
|
17
|
+
# Fetch the target page through a residential proxy in this country (ISO 3166-1
|
|
18
|
+
# alpha-2).
|
|
20
19
|
#
|
|
21
20
|
# @return [Symbol, ContextDev::Models::WebWebCrawlMdParams::Country, nil]
|
|
22
21
|
optional :country, enum: -> { ContextDev::WebWebCrawlMdParams::Country }
|
|
@@ -164,7 +163,7 @@ module ContextDev
|
|
|
164
163
|
#
|
|
165
164
|
# @param url [String] The starting URL for the crawl (must include http:// or https:// protocol)
|
|
166
165
|
#
|
|
167
|
-
# @param country [Symbol, ContextDev::Models::WebWebCrawlMdParams::Country]
|
|
166
|
+
# @param country [Symbol, ContextDev::Models::WebWebCrawlMdParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al
|
|
168
167
|
#
|
|
169
168
|
# @param exclude_selectors [Array<String>] CSS selectors to remove before each crawled page is converted to Markdown. Appli
|
|
170
169
|
#
|
|
@@ -206,9 +205,8 @@ module ContextDev
|
|
|
206
205
|
#
|
|
207
206
|
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}]
|
|
208
207
|
|
|
209
|
-
#
|
|
210
|
-
#
|
|
211
|
-
# countries. When provided, Context.dev fetches the target page from that country.
|
|
208
|
+
# Fetch the target page through a residential proxy in this country (ISO 3166-1
|
|
209
|
+
# alpha-2).
|
|
212
210
|
module Country
|
|
213
211
|
extend ContextDev::Internal::Type::Enum
|
|
214
212
|
|
|
@@ -24,9 +24,8 @@ module ContextDev
|
|
|
24
24
|
nil?: true
|
|
25
25
|
|
|
26
26
|
# @!attribute country
|
|
27
|
-
#
|
|
28
|
-
#
|
|
29
|
-
# countries. When provided, Context.dev fetches the target page from that country.
|
|
27
|
+
# Fetch the target page through a residential proxy in this country (ISO 3166-1
|
|
28
|
+
# alpha-2).
|
|
30
29
|
#
|
|
31
30
|
# @return [Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Country, nil]
|
|
32
31
|
optional :country, enum: -> { ContextDev::WebWebScrapeHTMLParams::Country }
|
|
@@ -131,7 +130,7 @@ module ContextDev
|
|
|
131
130
|
#
|
|
132
131
|
# @param actions [Array<ContextDev::Models::WebWebScrapeHTMLParams::Action::Wait, ContextDev::Models::WebWebScrapeHTMLParams::Action::Perform>, nil] Optional browser actions executed in array order after the page loads and before
|
|
133
132
|
#
|
|
134
|
-
# @param country [Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Country]
|
|
133
|
+
# @param country [Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al
|
|
135
134
|
#
|
|
136
135
|
# @param exclude_selectors [Array<String>, nil] CSS selectors to remove from the result. Applied after includeSelectors. Exclusi
|
|
137
136
|
#
|
|
@@ -212,9 +211,8 @@ module ContextDev
|
|
|
212
211
|
# @return [Array(ContextDev::Models::WebWebScrapeHTMLParams::Action::Wait, ContextDev::Models::WebWebScrapeHTMLParams::Action::Perform)]
|
|
213
212
|
end
|
|
214
213
|
|
|
215
|
-
#
|
|
216
|
-
#
|
|
217
|
-
# countries. When provided, Context.dev fetches the target page from that country.
|
|
214
|
+
# Fetch the target page through a residential proxy in this country (ISO 3166-1
|
|
215
|
+
# alpha-2).
|
|
218
216
|
module Country
|
|
219
217
|
extend ContextDev::Internal::Type::Enum
|
|
220
218
|
|
|
@@ -25,9 +25,8 @@ module ContextDev
|
|
|
25
25
|
nil?: true
|
|
26
26
|
|
|
27
27
|
# @!attribute country
|
|
28
|
-
#
|
|
29
|
-
#
|
|
30
|
-
# countries. When provided, Context.dev fetches the target page from that country.
|
|
28
|
+
# Fetch the target page through a residential proxy in this country (ISO 3166-1
|
|
29
|
+
# alpha-2).
|
|
31
30
|
#
|
|
32
31
|
# @return [Symbol, ContextDev::Models::WebWebScrapeMdParams::Country, nil]
|
|
33
32
|
optional :country, enum: -> { ContextDev::WebWebScrapeMdParams::Country }
|
|
@@ -150,7 +149,7 @@ module ContextDev
|
|
|
150
149
|
#
|
|
151
150
|
# @param actions [Array<ContextDev::Models::WebWebScrapeMdParams::Action::Wait, ContextDev::Models::WebWebScrapeMdParams::Action::Perform>, nil] Optional browser actions executed in array order after the page loads and before
|
|
152
151
|
#
|
|
153
|
-
# @param country [Symbol, ContextDev::Models::WebWebScrapeMdParams::Country]
|
|
152
|
+
# @param country [Symbol, ContextDev::Models::WebWebScrapeMdParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al
|
|
154
153
|
#
|
|
155
154
|
# @param exclude_selectors [Array<String>, nil] CSS selectors to remove before conversion to Markdown. Applied after includeSele
|
|
156
155
|
#
|
|
@@ -237,9 +236,8 @@ module ContextDev
|
|
|
237
236
|
# @return [Array(ContextDev::Models::WebWebScrapeMdParams::Action::Wait, ContextDev::Models::WebWebScrapeMdParams::Action::Perform)]
|
|
238
237
|
end
|
|
239
238
|
|
|
240
|
-
#
|
|
241
|
-
#
|
|
242
|
-
# countries. When provided, Context.dev fetches the target page from that country.
|
|
239
|
+
# Fetch the target page through a residential proxy in this country (ISO 3166-1
|
|
240
|
+
# alpha-2).
|
|
243
241
|
module Country
|
|
244
242
|
extend ContextDev::Internal::Type::Enum
|
|
245
243
|
|
data/lib/context_dev/models.rb
CHANGED
|
@@ -57,14 +57,18 @@ module ContextDev
|
|
|
57
57
|
|
|
58
58
|
BrandRetrieveSimplifiedParams = ContextDev::Models::BrandRetrieveSimplifiedParams
|
|
59
59
|
|
|
60
|
-
|
|
60
|
+
BrandSearchParams = ContextDev::Models::BrandSearchParams
|
|
61
61
|
|
|
62
|
-
|
|
62
|
+
CrawlControls = ContextDev::Models::CrawlControls
|
|
63
|
+
|
|
64
|
+
Failure = ContextDev::Models::Failure
|
|
63
65
|
|
|
64
66
|
IndustryRetrieveNaicsParams = ContextDev::Models::IndustryRetrieveNaicsParams
|
|
65
67
|
|
|
66
68
|
IndustryRetrieveSicParams = ContextDev::Models::IndustryRetrieveSicParams
|
|
67
69
|
|
|
70
|
+
Intake = ContextDev::Models::Intake
|
|
71
|
+
|
|
68
72
|
MonitorCreateParams = ContextDev::Models::MonitorCreateParams
|
|
69
73
|
|
|
70
74
|
MonitorDeleteParams = ContextDev::Models::MonitorDeleteParams
|
|
@@ -91,6 +95,8 @@ module ContextDev
|
|
|
91
95
|
|
|
92
96
|
MonitorUpdateParams = ContextDev::Models::MonitorUpdateParams
|
|
93
97
|
|
|
98
|
+
PageErrorCount = ContextDev::Models::PageErrorCount
|
|
99
|
+
|
|
94
100
|
ParseHandleParams = ContextDev::Models::ParseHandleParams
|
|
95
101
|
|
|
96
102
|
UtilityPrefetchParams = ContextDev::Models::UtilityPrefetchParams
|
|
@@ -3,9 +3,7 @@
|
|
|
3
3
|
module ContextDev
|
|
4
4
|
module Resources
|
|
5
5
|
class Batch
|
|
6
|
-
# Check progress and get download links
|
|
7
|
-
# rejected-URL list and webhook signing secret from submission, so nothing is lost
|
|
8
|
-
# if the submit response was dropped.
|
|
6
|
+
# Check progress, and get download links once the batch finishes.
|
|
9
7
|
#
|
|
10
8
|
# @overload retrieve(batch_id, request_options: {})
|
|
11
9
|
#
|
|
@@ -86,9 +84,8 @@ module ContextDev
|
|
|
86
84
|
# Some parameter documentations has been truncated, see
|
|
87
85
|
# {ContextDev::Models::BatchGetResultsParams} for more details.
|
|
88
86
|
#
|
|
89
|
-
# Page through
|
|
90
|
-
#
|
|
91
|
-
# the NDJSON files yourself.
|
|
87
|
+
# Page through a finished batch's results as JSON instead of downloading the
|
|
88
|
+
# NDJSON files.
|
|
92
89
|
#
|
|
93
90
|
# @overload get_results(batch_id, cursor: nil, limit: nil, request_options: {})
|
|
94
91
|
#
|
|
@@ -65,6 +65,40 @@ module ContextDev
|
|
|
65
65
|
)
|
|
66
66
|
end
|
|
67
67
|
|
|
68
|
+
# Some parameter documentations has been truncated, see
|
|
69
|
+
# {ContextDev::Models::BrandSearchParams} for more details.
|
|
70
|
+
#
|
|
71
|
+
# Search brands by name or domain and get back up to 10 lightweight matches
|
|
72
|
+
# (domain, name, logo), most popular first: by Tranco rank, then market cap for
|
|
73
|
+
# brands outside the Tranco list, with text relevance breaking ties. Matching is
|
|
74
|
+
# prefix-based with no typo tolerance, so it is suited to autocomplete. Only
|
|
75
|
+
# brands already in the Context.dev index are returned — use /brand/retrieve to
|
|
76
|
+
# fetch (and index) a specific domain. Free on Pro and Scale plans; costs 1 credit
|
|
77
|
+
# per request on the Free and Starter plans.
|
|
78
|
+
#
|
|
79
|
+
# @overload search(query:, tags: nil, request_options: {})
|
|
80
|
+
#
|
|
81
|
+
# @param query [String] Search term, matched against brand names and domains by prefix (e.g. 'nike', 'ni
|
|
82
|
+
#
|
|
83
|
+
# @param tags [Array<String>] Optional comma-separated caller-defined tags for tracking this request. Tags are
|
|
84
|
+
#
|
|
85
|
+
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}, nil]
|
|
86
|
+
#
|
|
87
|
+
# @return [ContextDev::Models::BrandSearchResponse]
|
|
88
|
+
#
|
|
89
|
+
# @see ContextDev::Models::BrandSearchParams
|
|
90
|
+
def search(params)
|
|
91
|
+
parsed, options = ContextDev::BrandSearchParams.dump_request(params)
|
|
92
|
+
query = ContextDev::Internal::Util.encode_query_params(parsed)
|
|
93
|
+
@client.request(
|
|
94
|
+
method: :get,
|
|
95
|
+
path: "brand/search",
|
|
96
|
+
query: query,
|
|
97
|
+
model: ContextDev::Models::BrandSearchResponse,
|
|
98
|
+
options: options
|
|
99
|
+
)
|
|
100
|
+
end
|
|
101
|
+
|
|
68
102
|
# @api private
|
|
69
103
|
#
|
|
70
104
|
# @param client [ContextDev::Client]
|
|
@@ -180,7 +180,7 @@ module ContextDev
|
|
|
180
180
|
#
|
|
181
181
|
# @param color_scheme [Symbol, ContextDev::Models::WebScreenshotParams::ColorScheme] Optional parameter to choose the site's visual theme in the screenshot. Use 'lig
|
|
182
182
|
#
|
|
183
|
-
# @param country [Symbol, ContextDev::Models::WebScreenshotParams::Country]
|
|
183
|
+
# @param country [Symbol, ContextDev::Models::WebScreenshotParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al
|
|
184
184
|
#
|
|
185
185
|
# @param direct_url [String] A specific URL to screenshot directly, bypassing domain resolution (e.g., 'https
|
|
186
186
|
#
|
|
@@ -285,7 +285,7 @@ module ContextDev
|
|
|
285
285
|
#
|
|
286
286
|
# @param url [String] The starting URL for the crawl (must include http:// or https:// protocol)
|
|
287
287
|
#
|
|
288
|
-
# @param country [Symbol, ContextDev::Models::WebWebCrawlMdParams::Country]
|
|
288
|
+
# @param country [Symbol, ContextDev::Models::WebWebCrawlMdParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al
|
|
289
289
|
#
|
|
290
290
|
# @param exclude_selectors [Array<String>] CSS selectors to remove before each crawled page is converted to Markdown. Appli
|
|
291
291
|
#
|
|
@@ -353,7 +353,7 @@ module ContextDev
|
|
|
353
353
|
#
|
|
354
354
|
# @param actions [Array<ContextDev::Models::WebWebScrapeHTMLParams::Action::Wait, ContextDev::Models::WebWebScrapeHTMLParams::Action::Perform>, nil] Optional browser actions executed in array order after the page loads and before
|
|
355
355
|
#
|
|
356
|
-
# @param country [Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Country]
|
|
356
|
+
# @param country [Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al
|
|
357
357
|
#
|
|
358
358
|
# @param exclude_selectors [Array<String>, nil] CSS selectors to remove from the result. Applied after includeSelectors. Exclusi
|
|
359
359
|
#
|
|
@@ -481,7 +481,7 @@ module ContextDev
|
|
|
481
481
|
#
|
|
482
482
|
# @param actions [Array<ContextDev::Models::WebWebScrapeMdParams::Action::Wait, ContextDev::Models::WebWebScrapeMdParams::Action::Perform>, nil] Optional browser actions executed in array order after the page loads and before
|
|
483
483
|
#
|
|
484
|
-
# @param country [Symbol, ContextDev::Models::WebWebScrapeMdParams::Country]
|
|
484
|
+
# @param country [Symbol, ContextDev::Models::WebWebScrapeMdParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al
|
|
485
485
|
#
|
|
486
486
|
# @param exclude_selectors [Array<String>, nil] CSS selectors to remove before conversion to Markdown. Applied after includeSele
|
|
487
487
|
#
|
data/lib/context_dev/version.rb
CHANGED
data/lib/context_dev.rb
CHANGED
|
@@ -70,12 +70,15 @@ require_relative "context_dev/models/brand_retrieve_params"
|
|
|
70
70
|
require_relative "context_dev/models/brand_retrieve_response"
|
|
71
71
|
require_relative "context_dev/models/brand_retrieve_simplified_params"
|
|
72
72
|
require_relative "context_dev/models/brand_retrieve_simplified_response"
|
|
73
|
-
require_relative "context_dev/models/
|
|
74
|
-
require_relative "context_dev/models/
|
|
73
|
+
require_relative "context_dev/models/brand_search_params"
|
|
74
|
+
require_relative "context_dev/models/brand_search_response"
|
|
75
|
+
require_relative "context_dev/models/crawl_controls"
|
|
76
|
+
require_relative "context_dev/models/failure"
|
|
75
77
|
require_relative "context_dev/models/industry_retrieve_naics_params"
|
|
76
78
|
require_relative "context_dev/models/industry_retrieve_naics_response"
|
|
77
79
|
require_relative "context_dev/models/industry_retrieve_sic_params"
|
|
78
80
|
require_relative "context_dev/models/industry_retrieve_sic_response"
|
|
81
|
+
require_relative "context_dev/models/intake"
|
|
79
82
|
require_relative "context_dev/models/monitor_create_params"
|
|
80
83
|
require_relative "context_dev/models/monitor_create_response"
|
|
81
84
|
require_relative "context_dev/models/monitor_delete_params"
|
|
@@ -102,6 +105,7 @@ require_relative "context_dev/models/monitor_run_params"
|
|
|
102
105
|
require_relative "context_dev/models/monitor_run_response"
|
|
103
106
|
require_relative "context_dev/models/monitor_update_params"
|
|
104
107
|
require_relative "context_dev/models/monitor_update_response"
|
|
108
|
+
require_relative "context_dev/models/page_error_count"
|
|
105
109
|
require_relative "context_dev/models/parse_handle_params"
|
|
106
110
|
require_relative "context_dev/models/parse_handle_response"
|
|
107
111
|
require_relative "context_dev/models/utility_prefetch_params"
|