context.dev 2.8.0 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +15 -0
- data/README.md +1 -1
- data/lib/context_dev/client.rb +5 -0
- data/lib/context_dev/models/batch_delete_params.rb +22 -0
- data/lib/context_dev/models/batch_delete_response.rb +60 -0
- data/lib/context_dev/models/batch_get_results_response.rb +13 -1
- data/lib/context_dev/models/batch_list_response.rb +13 -4
- data/lib/context_dev/models/batch_retrieve_response.rb +13 -4
- data/lib/context_dev/models/batch_submit_params.rb +2352 -26
- data/lib/context_dev/models/batch_submit_response.rb +125 -531
- data/lib/context_dev/models/brand_retrieve_response.rb +50 -1
- data/lib/context_dev/models/brand_search_response.rb +3 -2
- data/lib/context_dev/models/crawl_controls.rb +21 -15
- data/lib/context_dev/models/parse_handle_params.rb +11 -9
- data/lib/context_dev/models/person_enrich_params.rb +176 -0
- data/lib/context_dev/models/person_enrich_response.rb +581 -0
- data/lib/context_dev/models/web_search_response.rb +1 -0
- data/lib/context_dev/models/web_web_crawl_md_params.rb +5 -4
- data/lib/context_dev/models/web_web_scrape_html_params.rb +11 -9
- data/lib/context_dev/models/web_web_scrape_md_params.rb +11 -9
- data/lib/context_dev/models/web_web_scrape_sitemap_params.rb +11 -1
- data/lib/context_dev/models/web_web_scrape_sitemap_response.rb +3 -2
- data/lib/context_dev/models.rb +4 -0
- data/lib/context_dev/resources/batch.rb +33 -7
- data/lib/context_dev/resources/brand.rb +2 -1
- data/lib/context_dev/resources/parse.rb +1 -1
- data/lib/context_dev/resources/people.rb +56 -0
- data/lib/context_dev/resources/web.rb +21 -2
- data/lib/context_dev/version.rb +1 -1
- data/lib/context_dev.rb +5 -0
- data/rbi/context_dev/client.rbi +4 -0
- data/rbi/context_dev/models/batch_delete_params.rbi +40 -0
- data/rbi/context_dev/models/batch_delete_response.rbi +116 -0
- data/rbi/context_dev/models/batch_get_results_response.rbi +14 -1
- data/rbi/context_dev/models/batch_list_response.rbi +24 -8
- data/rbi/context_dev/models/batch_retrieve_response.rbi +24 -8
- data/rbi/context_dev/models/batch_submit_params.rbi +6955 -44
- data/rbi/context_dev/models/batch_submit_response.rbi +184 -1151
- data/rbi/context_dev/models/brand_retrieve_response.rbi +152 -0
- data/rbi/context_dev/models/brand_search_response.rbi +4 -2
- data/rbi/context_dev/models/crawl_controls.rbi +22 -28
- data/rbi/context_dev/models/parse_handle_params.rbi +15 -12
- data/rbi/context_dev/models/person_enrich_params.rbi +332 -0
- data/rbi/context_dev/models/person_enrich_response.rbi +1208 -0
- data/rbi/context_dev/models/web_search_response.rbi +5 -0
- data/rbi/context_dev/models/web_web_crawl_md_params.rbi +8 -6
- data/rbi/context_dev/models/web_web_scrape_html_params.rbi +15 -12
- data/rbi/context_dev/models/web_web_scrape_md_params.rbi +15 -12
- data/rbi/context_dev/models/web_web_scrape_sitemap_params.rbi +15 -0
- data/rbi/context_dev/models/web_web_scrape_sitemap_response.rbi +4 -2
- data/rbi/context_dev/models.rbi +4 -0
- data/rbi/context_dev/resources/batch.rbi +32 -10
- data/rbi/context_dev/resources/brand.rbi +2 -1
- data/rbi/context_dev/resources/parse.rbi +5 -5
- data/rbi/context_dev/resources/people.rbi +47 -0
- data/rbi/context_dev/resources/web.rbi +23 -1
- data/sig/context_dev/client.rbs +2 -0
- data/sig/context_dev/models/batch_delete_params.rbs +23 -0
- data/sig/context_dev/models/batch_delete_response.rbs +57 -0
- data/sig/context_dev/models/batch_get_results_response.rbs +9 -2
- data/sig/context_dev/models/batch_list_response.rbs +16 -2
- data/sig/context_dev/models/batch_retrieve_response.rbs +16 -2
- data/sig/context_dev/models/batch_submit_params.rbs +2756 -15
- data/sig/context_dev/models/batch_submit_response.rbs +78 -466
- data/sig/context_dev/models/brand_retrieve_response.rbs +62 -0
- data/sig/context_dev/models/crawl_controls.rbs +16 -16
- data/sig/context_dev/models/person_enrich_params.rbs +199 -0
- data/sig/context_dev/models/person_enrich_response.rbs +607 -0
- data/sig/context_dev/models/web_search_response.rbs +2 -0
- data/sig/context_dev/models/web_web_scrape_sitemap_params.rbs +7 -0
- data/sig/context_dev/models.rbs +4 -0
- data/sig/context_dev/resources/batch.rbs +8 -2
- data/sig/context_dev/resources/people.rbs +19 -0
- data/sig/context_dev/resources/web.rbs +1 -0
- metadata +17 -2
|
@@ -534,16 +534,17 @@ module ContextDev
|
|
|
534
534
|
optional :end_, Integer, api_name: :end
|
|
535
535
|
|
|
536
536
|
# @!attribute ocr
|
|
537
|
-
# When true,
|
|
538
|
-
#
|
|
539
|
-
#
|
|
537
|
+
# When true, OCR the selected PDF pages that have no usable text layer (scans),
|
|
538
|
+
# replacing each recovered page's text with the OCR result while pages with a real
|
|
539
|
+
# text layer keep it. Billed at 1 credit per page OCR actually recovered, on top
|
|
540
|
+
# of the base request cost. When false, no OCR runs.
|
|
540
541
|
#
|
|
541
542
|
# @return [Boolean, Symbol, ContextDev::Models::WebWebScrapeMdParams::Pdf::Ocr, nil]
|
|
542
543
|
optional :ocr, union: -> { ContextDev::WebWebScrapeMdParams::Pdf::Ocr }
|
|
543
544
|
|
|
544
545
|
# @!attribute should_parse
|
|
545
546
|
# When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and
|
|
546
|
-
# a 400
|
|
547
|
+
# a 400 PDF_SKIPPED is returned.
|
|
547
548
|
#
|
|
548
549
|
# @return [Boolean, Symbol, ContextDev::Models::WebWebScrapeMdParams::Pdf::ShouldParse, nil]
|
|
549
550
|
optional :should_parse,
|
|
@@ -565,15 +566,16 @@ module ContextDev
|
|
|
565
566
|
#
|
|
566
567
|
# @param end_ [Integer] Last 1-based PDF page to parse. When omitted, parsing ends at the last page. Mus
|
|
567
568
|
#
|
|
568
|
-
# @param ocr [Boolean, Symbol, ContextDev::Models::WebWebScrapeMdParams::Pdf::Ocr] When true,
|
|
569
|
+
# @param ocr [Boolean, Symbol, ContextDev::Models::WebWebScrapeMdParams::Pdf::Ocr] When true, OCR the selected PDF pages that have no usable text layer (scans), re
|
|
569
570
|
#
|
|
570
571
|
# @param should_parse [Boolean, Symbol, ContextDev::Models::WebWebScrapeMdParams::Pdf::ShouldParse] When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and
|
|
571
572
|
#
|
|
572
573
|
# @param start [Integer] First 1-based PDF page to parse. When omitted, parsing starts at the first page.
|
|
573
574
|
|
|
574
|
-
# When true,
|
|
575
|
-
#
|
|
576
|
-
#
|
|
575
|
+
# When true, OCR the selected PDF pages that have no usable text layer (scans),
|
|
576
|
+
# replacing each recovered page's text with the OCR result while pages with a real
|
|
577
|
+
# text layer keep it. Billed at 1 credit per page OCR actually recovered, on top
|
|
578
|
+
# of the base request cost. When false, no OCR runs.
|
|
577
579
|
#
|
|
578
580
|
# @see ContextDev::Models::WebWebScrapeMdParams::Pdf#ocr
|
|
579
581
|
module Ocr
|
|
@@ -601,7 +603,7 @@ module ContextDev
|
|
|
601
603
|
end
|
|
602
604
|
|
|
603
605
|
# When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and
|
|
604
|
-
# a 400
|
|
606
|
+
# a 400 PDF_SKIPPED is returned.
|
|
605
607
|
#
|
|
606
608
|
# @see ContextDev::Models::WebWebScrapeMdParams::Pdf#should_parse
|
|
607
609
|
module ShouldParse
|
|
@@ -28,6 +28,14 @@ module ContextDev
|
|
|
28
28
|
# @return [Integer, nil]
|
|
29
29
|
optional :max_links, Integer
|
|
30
30
|
|
|
31
|
+
# @!attribute search
|
|
32
|
+
# Optional search phrase. When provided, the crawled sitemap is filtered to the
|
|
33
|
+
# pages whose URLs are about that phrase, most relevant first, and the request
|
|
34
|
+
# costs 2 credits instead of 1.
|
|
35
|
+
#
|
|
36
|
+
# @return [String, nil]
|
|
37
|
+
optional :search, String
|
|
38
|
+
|
|
31
39
|
# @!attribute sitemap_url
|
|
32
40
|
# Optional explicit sitemap URL. When provided, exactly this sitemap is crawled
|
|
33
41
|
# instead of discovering the domain's sitemaps.
|
|
@@ -67,7 +75,7 @@ module ContextDev
|
|
|
67
75
|
# @return [Symbol, ContextDev::Models::WebWebScrapeSitemapParams::Zdr, nil]
|
|
68
76
|
optional :zdr, enum: -> { ContextDev::WebWebScrapeSitemapParams::Zdr }
|
|
69
77
|
|
|
70
|
-
# @!method initialize(domain:, headers: nil, max_links: nil, sitemap_url: nil, tags: nil, timeout_ms: nil, url_regex: nil, zdr: nil, request_options: {})
|
|
78
|
+
# @!method initialize(domain:, headers: nil, max_links: nil, search: nil, sitemap_url: nil, tags: nil, timeout_ms: nil, url_regex: nil, zdr: nil, request_options: {})
|
|
71
79
|
# Some parameter documentations has been truncated, see
|
|
72
80
|
# {ContextDev::Models::WebWebScrapeSitemapParams} for more details.
|
|
73
81
|
#
|
|
@@ -77,6 +85,8 @@ module ContextDev
|
|
|
77
85
|
#
|
|
78
86
|
# @param max_links [Integer] Maximum number of links to return from the sitemap crawl. Defaults to 10,000. Mi
|
|
79
87
|
#
|
|
88
|
+
# @param search [String] Optional search phrase. When provided, the crawled sitemap is filtered to the pa
|
|
89
|
+
#
|
|
80
90
|
# @param sitemap_url [String] Optional explicit sitemap URL. When provided, exactly this sitemap is crawled in
|
|
81
91
|
#
|
|
82
92
|
# @param tags [Array<String>] Optional comma-separated caller-defined tags for tracking this request. Tags are
|
|
@@ -23,7 +23,8 @@ module ContextDev
|
|
|
23
23
|
required :success, enum: -> { ContextDev::Models::WebWebScrapeSitemapResponse::Success }
|
|
24
24
|
|
|
25
25
|
# @!attribute urls
|
|
26
|
-
#
|
|
26
|
+
# Discovered page URLs from the sitemap, up to `maxLinks`. When `search` is set
|
|
27
|
+
# these are only the matching pages, most relevant first.
|
|
27
28
|
#
|
|
28
29
|
# @return [Array<String>]
|
|
29
30
|
required :urls, ContextDev::Internal::Type::ArrayOf[String]
|
|
@@ -45,7 +46,7 @@ module ContextDev
|
|
|
45
46
|
#
|
|
46
47
|
# @param success [Boolean, ContextDev::Models::WebWebScrapeSitemapResponse::Success] Indicates success
|
|
47
48
|
#
|
|
48
|
-
# @param urls [Array<String>]
|
|
49
|
+
# @param urls [Array<String>] Discovered page URLs from the sitemap, up to `maxLinks`. When `search` is set th
|
|
49
50
|
#
|
|
50
51
|
# @param key_metadata [ContextDev::Models::WebWebScrapeSitemapResponse::KeyMetadata] Metadata about the API key used for the request. Included in every response when
|
|
51
52
|
|
data/lib/context_dev/models.rb
CHANGED
|
@@ -45,6 +45,8 @@ module ContextDev
|
|
|
45
45
|
|
|
46
46
|
BatchCancelParams = ContextDev::Models::BatchCancelParams
|
|
47
47
|
|
|
48
|
+
BatchDeleteParams = ContextDev::Models::BatchDeleteParams
|
|
49
|
+
|
|
48
50
|
BatchGetResultsParams = ContextDev::Models::BatchGetResultsParams
|
|
49
51
|
|
|
50
52
|
BatchListParams = ContextDev::Models::BatchListParams
|
|
@@ -99,6 +101,8 @@ module ContextDev
|
|
|
99
101
|
|
|
100
102
|
ParseHandleParams = ContextDev::Models::ParseHandleParams
|
|
101
103
|
|
|
104
|
+
PersonEnrichParams = ContextDev::Models::PersonEnrichParams
|
|
105
|
+
|
|
102
106
|
UtilityPrefetchParams = ContextDev::Models::UtilityPrefetchParams
|
|
103
107
|
|
|
104
108
|
WebExtractCompetitorsParams = ContextDev::Models::WebExtractCompetitorsParams
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
module ContextDev
|
|
4
4
|
module Resources
|
|
5
|
+
# Scrape many pages or crawl a site asynchronously.
|
|
5
6
|
class Batch
|
|
6
7
|
# Check progress, and get download links once the batch finishes.
|
|
7
8
|
#
|
|
@@ -60,6 +61,27 @@ module ContextDev
|
|
|
60
61
|
)
|
|
61
62
|
end
|
|
62
63
|
|
|
64
|
+
# Permanently delete a finished batch and its stored results. Active batches must
|
|
65
|
+
# settle first.
|
|
66
|
+
#
|
|
67
|
+
# @overload delete(batch_id, request_options: {})
|
|
68
|
+
#
|
|
69
|
+
# @param batch_id [String] ID of the batch to retrieve or cancel.
|
|
70
|
+
#
|
|
71
|
+
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}, nil]
|
|
72
|
+
#
|
|
73
|
+
# @return [ContextDev::Models::BatchDeleteResponse]
|
|
74
|
+
#
|
|
75
|
+
# @see ContextDev::Models::BatchDeleteParams
|
|
76
|
+
def delete(batch_id, params = {})
|
|
77
|
+
@client.request(
|
|
78
|
+
method: :delete,
|
|
79
|
+
path: ["batch/%1$s", batch_id],
|
|
80
|
+
model: ContextDev::Models::BatchDeleteResponse,
|
|
81
|
+
options: params[:request_options]
|
|
82
|
+
)
|
|
83
|
+
end
|
|
84
|
+
|
|
63
85
|
# Stop a batch from starting new pages. In-progress pages finish, and unused
|
|
64
86
|
# credits are refunded.
|
|
65
87
|
#
|
|
@@ -115,15 +137,17 @@ module ContextDev
|
|
|
115
137
|
# Some parameter documentations has been truncated, see
|
|
116
138
|
# {ContextDev::Models::BatchSubmitParams} for more details.
|
|
117
139
|
#
|
|
118
|
-
#
|
|
140
|
+
# Scrape 25K URLs or crawl large websites asynchronously.
|
|
141
|
+
#
|
|
142
|
+
# @overload submit(input:, tags: nil, webhook_url: nil, idempotency_key: nil, request_options: {})
|
|
119
143
|
#
|
|
120
|
-
# @
|
|
144
|
+
# @param input [ContextDev::Models::BatchSubmitParams::Input::Scrape, ContextDev::Models::BatchSubmitParams::Input::Crawl] Body param: Choose a URL list or a site crawl.
|
|
121
145
|
#
|
|
122
|
-
# @param
|
|
146
|
+
# @param tags [Array<String>] Body param: Tags stored on the batch. Filter the batch list by them later.
|
|
123
147
|
#
|
|
124
|
-
# @param
|
|
148
|
+
# @param webhook_url [String] Body param: URL notified when the batch finishes.
|
|
125
149
|
#
|
|
126
|
-
# @param
|
|
150
|
+
# @param idempotency_key [String] Header param: Any string unique to this submission. Retries with the same key re
|
|
127
151
|
#
|
|
128
152
|
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}, nil]
|
|
129
153
|
#
|
|
@@ -132,10 +156,12 @@ module ContextDev
|
|
|
132
156
|
# @see ContextDev::Models::BatchSubmitParams
|
|
133
157
|
def submit(params)
|
|
134
158
|
parsed, options = ContextDev::BatchSubmitParams.dump_request(params)
|
|
159
|
+
header_params = {idempotency_key: "idempotency-key"}
|
|
135
160
|
@client.request(
|
|
136
161
|
method: :post,
|
|
137
|
-
path: "
|
|
138
|
-
|
|
162
|
+
path: "batch/submit",
|
|
163
|
+
headers: parsed.slice(*header_params.keys).transform_keys(header_params),
|
|
164
|
+
body: parsed.except(*header_params.keys),
|
|
139
165
|
model: ContextDev::Models::BatchSubmitResponse,
|
|
140
166
|
options: options
|
|
141
167
|
)
|
|
@@ -69,7 +69,8 @@ module ContextDev
|
|
|
69
69
|
# {ContextDev::Models::BrandSearchParams} for more details.
|
|
70
70
|
#
|
|
71
71
|
# Search brands by name or domain and get back up to 10 lightweight matches
|
|
72
|
-
# (domain, name, logo)
|
|
72
|
+
# (domain, name, logo). Name matches rank ahead of domain matches; within each
|
|
73
|
+
# group the most popular brands come first: by Tranco rank, then market cap for
|
|
73
74
|
# brands outside the Tranco list, with text relevance breaking ties. Matching is
|
|
74
75
|
# prefix-based with no typo tolerance, so it is suited to autocomplete. Only
|
|
75
76
|
# brands already in the Context.dev index are returned — use /brand/retrieve to
|
|
@@ -21,7 +21,7 @@ module ContextDev
|
|
|
21
21
|
#
|
|
22
22
|
# @param include_links [Boolean, Symbol, ContextDev::Models::ParseHandleParams::IncludeLinks] Query param: Preserve hyperlinks in Markdown output
|
|
23
23
|
#
|
|
24
|
-
# @param ocr [Boolean, Symbol, ContextDev::Models::ParseHandleParams::Ocr] Query param: When true for PDF inputs,
|
|
24
|
+
# @param ocr [Boolean, Symbol, ContextDev::Models::ParseHandleParams::Ocr] Query param: When true for PDF inputs, OCR the selected pages that have no usabl
|
|
25
25
|
#
|
|
26
26
|
# @param pdf [ContextDev::Models::ParseHandleParams::Pdf] Query param: PDF page-range options as a JSON object, e.g. {"start": 2, "end": 5
|
|
27
27
|
#
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ContextDev
|
|
4
|
+
module Resources
|
|
5
|
+
class People
|
|
6
|
+
# Some parameter documentations has been truncated, see
|
|
7
|
+
# {ContextDev::Models::PersonEnrichParams} for more details.
|
|
8
|
+
#
|
|
9
|
+
# Finds and normalizes the best available person candidate from additive identity
|
|
10
|
+
# clues, then assigns an identity match score from 0 to 100. Available on all paid
|
|
11
|
+
# plans. Successful requests cost 20 credits. Disposable and free email addresses
|
|
12
|
+
# (like gmail.com, yahoo.com) will throw a 422 error.
|
|
13
|
+
#
|
|
14
|
+
# @overload enrich(company: nil, education: nil, email: nil, location: nil, name: nil, social_urls: nil, tags: nil, timeout_ms: nil, request_options: {})
|
|
15
|
+
#
|
|
16
|
+
# @param company [ContextDev::Models::PersonEnrichParams::Company]
|
|
17
|
+
#
|
|
18
|
+
# @param education [Array<ContextDev::Models::PersonEnrichParams::Education>]
|
|
19
|
+
#
|
|
20
|
+
# @param email [String]
|
|
21
|
+
#
|
|
22
|
+
# @param location [ContextDev::Models::PersonEnrichParams::Location]
|
|
23
|
+
#
|
|
24
|
+
# @param name [ContextDev::Models::PersonEnrichParams::Name]
|
|
25
|
+
#
|
|
26
|
+
# @param social_urls [Array<String>]
|
|
27
|
+
#
|
|
28
|
+
# @param tags [Array<String>] Optional tags for tracking usage. Up to 20 tags, each 1 to 50 characters.
|
|
29
|
+
#
|
|
30
|
+
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
31
|
+
#
|
|
32
|
+
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}, nil]
|
|
33
|
+
#
|
|
34
|
+
# @return [ContextDev::Models::PersonEnrichResponse]
|
|
35
|
+
#
|
|
36
|
+
# @see ContextDev::Models::PersonEnrichParams
|
|
37
|
+
def enrich(params = {})
|
|
38
|
+
parsed, options = ContextDev::PersonEnrichParams.dump_request(params)
|
|
39
|
+
@client.request(
|
|
40
|
+
method: :post,
|
|
41
|
+
path: "people/enrich",
|
|
42
|
+
body: parsed,
|
|
43
|
+
model: ContextDev::Models::PersonEnrichResponse,
|
|
44
|
+
options: options
|
|
45
|
+
)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# @api private
|
|
49
|
+
#
|
|
50
|
+
# @param client [ContextDev::Client]
|
|
51
|
+
def initialize(client:)
|
|
52
|
+
@client = client
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
@@ -462,6 +462,18 @@ module ContextDev
|
|
|
462
462
|
# responses from a recognized API key; use error_code to distinguish stable
|
|
463
463
|
# failure categories.
|
|
464
464
|
#
|
|
465
|
+
# ### YouTube
|
|
466
|
+
#
|
|
467
|
+
# YouTube URLs return the video or channel itself rather than the surrounding
|
|
468
|
+
# player and navigation chrome. A URL addressing a single video (`/watch`,
|
|
469
|
+
# `youtu.be`, `/shorts`, `/embed`, `/live`) returns its title, channel, duration,
|
|
470
|
+
# view count, keywords, full description, and the transcript when the video has
|
|
471
|
+
# captions that can be retrieved; videos without captions return everything except
|
|
472
|
+
# the transcript. A channel URL (`/channel/UC…`, `/@handle`, `/c/…`, `/user/…`)
|
|
473
|
+
# returns its name, handle, subscriber count, video count, and full description.
|
|
474
|
+
# When `includeImages=true`, video responses also include the thumbnail and
|
|
475
|
+
# channel responses include the avatar. Costs the same as any other scrape.
|
|
476
|
+
#
|
|
465
477
|
# ### Billing & errors
|
|
466
478
|
#
|
|
467
479
|
# | HTTP status | Billed? | Meaning |
|
|
@@ -471,6 +483,7 @@ module ContextDev
|
|
|
471
483
|
# | 401 / 403 | No | Invalid/disabled key, insufficient permissions, or credits exhausted; inspect error_code |
|
|
472
484
|
# | 404 | No | Target page returned or fingerprinted as not found |
|
|
473
485
|
# | 408 | No | Request timed out |
|
|
486
|
+
# | 413 | No | Target content exceeds the maximum supported size (20 MB) |
|
|
474
487
|
# | 415 | No | Unsupported content type |
|
|
475
488
|
# | 429 | No | Per-minute rate limit exceeded; honor Retry-After |
|
|
476
489
|
# | 500 | No | Internal error |
|
|
@@ -545,9 +558,13 @@ module ContextDev
|
|
|
545
558
|
# Some parameter documentations has been truncated, see
|
|
546
559
|
# {ContextDev::Models::WebWebScrapeSitemapParams} for more details.
|
|
547
560
|
#
|
|
548
|
-
# Crawl an entire website's sitemap and return all discovered page URLs.
|
|
561
|
+
# Crawl an entire website's sitemap and return all discovered page URLs. Pass
|
|
562
|
+
# `search` to have the crawled sitemap filtered down to the pages about a phrase
|
|
563
|
+
# (for example `pricing and plans` or `api authentication docs`), most relevant
|
|
564
|
+
# first — a searched crawl scans the whole sitemap and costs 2 credits instead
|
|
565
|
+
# of 1.
|
|
549
566
|
#
|
|
550
|
-
# @overload web_scrape_sitemap(domain:, headers: nil, max_links: nil, sitemap_url: nil, tags: nil, timeout_ms: nil, url_regex: nil, zdr: nil, request_options: {})
|
|
567
|
+
# @overload web_scrape_sitemap(domain:, headers: nil, max_links: nil, search: nil, sitemap_url: nil, tags: nil, timeout_ms: nil, url_regex: nil, zdr: nil, request_options: {})
|
|
551
568
|
#
|
|
552
569
|
# @param domain [String] Domain to build a sitemap for
|
|
553
570
|
#
|
|
@@ -555,6 +572,8 @@ module ContextDev
|
|
|
555
572
|
#
|
|
556
573
|
# @param max_links [Integer] Maximum number of links to return from the sitemap crawl. Defaults to 10,000. Mi
|
|
557
574
|
#
|
|
575
|
+
# @param search [String] Optional search phrase. When provided, the crawled sitemap is filtered to the pa
|
|
576
|
+
#
|
|
558
577
|
# @param sitemap_url [String] Optional explicit sitemap URL. When provided, exactly this sitemap is crawled in
|
|
559
578
|
#
|
|
560
579
|
# @param tags [Array<String>] Optional comma-separated caller-defined tags for tracking this request. Tags are
|
data/lib/context_dev/version.rb
CHANGED
data/lib/context_dev.rb
CHANGED
|
@@ -58,6 +58,8 @@ require_relative "context_dev/models/ai_extract_products_params"
|
|
|
58
58
|
require_relative "context_dev/models/ai_extract_products_response"
|
|
59
59
|
require_relative "context_dev/models/batch_cancel_params"
|
|
60
60
|
require_relative "context_dev/models/batch_cancel_response"
|
|
61
|
+
require_relative "context_dev/models/batch_delete_params"
|
|
62
|
+
require_relative "context_dev/models/batch_delete_response"
|
|
61
63
|
require_relative "context_dev/models/batch_get_results_params"
|
|
62
64
|
require_relative "context_dev/models/batch_get_results_response"
|
|
63
65
|
require_relative "context_dev/models/batch_list_params"
|
|
@@ -108,6 +110,8 @@ require_relative "context_dev/models/monitor_update_response"
|
|
|
108
110
|
require_relative "context_dev/models/page_error_count"
|
|
109
111
|
require_relative "context_dev/models/parse_handle_params"
|
|
110
112
|
require_relative "context_dev/models/parse_handle_response"
|
|
113
|
+
require_relative "context_dev/models/person_enrich_params"
|
|
114
|
+
require_relative "context_dev/models/person_enrich_response"
|
|
111
115
|
require_relative "context_dev/models/utility_prefetch_params"
|
|
112
116
|
require_relative "context_dev/models/utility_prefetch_response"
|
|
113
117
|
require_relative "context_dev/models/web_extract_competitors_params"
|
|
@@ -140,5 +144,6 @@ require_relative "context_dev/resources/brand"
|
|
|
140
144
|
require_relative "context_dev/resources/industry"
|
|
141
145
|
require_relative "context_dev/resources/monitors"
|
|
142
146
|
require_relative "context_dev/resources/parse"
|
|
147
|
+
require_relative "context_dev/resources/people"
|
|
143
148
|
require_relative "context_dev/resources/utility"
|
|
144
149
|
require_relative "context_dev/resources/web"
|
data/rbi/context_dev/client.rbi
CHANGED
|
@@ -38,9 +38,13 @@ module ContextDev
|
|
|
38
38
|
sig { returns(ContextDev::Resources::Monitors) }
|
|
39
39
|
attr_reader :monitors
|
|
40
40
|
|
|
41
|
+
# Scrape many pages or crawl a site asynchronously.
|
|
41
42
|
sig { returns(ContextDev::Resources::Batch) }
|
|
42
43
|
attr_reader :batch
|
|
43
44
|
|
|
45
|
+
sig { returns(ContextDev::Resources::People) }
|
|
46
|
+
attr_reader :people
|
|
47
|
+
|
|
44
48
|
# @api private
|
|
45
49
|
sig { override.returns(T::Hash[String, String]) }
|
|
46
50
|
private def auth_headers
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# typed: strong
|
|
2
|
+
|
|
3
|
+
module ContextDev
|
|
4
|
+
module Models
|
|
5
|
+
class BatchDeleteParams < ContextDev::Internal::Type::BaseModel
|
|
6
|
+
extend ContextDev::Internal::Type::RequestParameters::Converter
|
|
7
|
+
include ContextDev::Internal::Type::RequestParameters
|
|
8
|
+
|
|
9
|
+
OrHash =
|
|
10
|
+
T.type_alias do
|
|
11
|
+
T.any(ContextDev::BatchDeleteParams, ContextDev::Internal::AnyHash)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# ID of the batch to retrieve or cancel.
|
|
15
|
+
sig { returns(String) }
|
|
16
|
+
attr_accessor :batch_id
|
|
17
|
+
|
|
18
|
+
sig do
|
|
19
|
+
params(
|
|
20
|
+
batch_id: String,
|
|
21
|
+
request_options: ContextDev::RequestOptions::OrHash
|
|
22
|
+
).returns(T.attached_class)
|
|
23
|
+
end
|
|
24
|
+
def self.new(
|
|
25
|
+
# ID of the batch to retrieve or cancel.
|
|
26
|
+
batch_id:,
|
|
27
|
+
request_options: {}
|
|
28
|
+
)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
sig do
|
|
32
|
+
override.returns(
|
|
33
|
+
{ batch_id: String, request_options: ContextDev::RequestOptions }
|
|
34
|
+
)
|
|
35
|
+
end
|
|
36
|
+
def to_hash
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
# typed: strong
|
|
2
|
+
|
|
3
|
+
module ContextDev
|
|
4
|
+
module Models
|
|
5
|
+
class BatchDeleteResponse < ContextDev::Internal::Type::BaseModel
|
|
6
|
+
OrHash =
|
|
7
|
+
T.type_alias do
|
|
8
|
+
T.any(
|
|
9
|
+
ContextDev::Models::BatchDeleteResponse,
|
|
10
|
+
ContextDev::Internal::AnyHash
|
|
11
|
+
)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# ID of the deleted batch.
|
|
15
|
+
sig { returns(T.nilable(String)) }
|
|
16
|
+
attr_reader :id
|
|
17
|
+
|
|
18
|
+
sig { params(id: String).void }
|
|
19
|
+
attr_writer :id
|
|
20
|
+
|
|
21
|
+
# Always true on success.
|
|
22
|
+
sig { returns(T.nilable(T::Boolean)) }
|
|
23
|
+
attr_reader :deleted
|
|
24
|
+
|
|
25
|
+
sig { params(deleted: T::Boolean).void }
|
|
26
|
+
attr_writer :deleted
|
|
27
|
+
|
|
28
|
+
# Metadata about the API key used for the request. Included in every response
|
|
29
|
+
# whenever a valid API key is provided, even when the response status is not 200.
|
|
30
|
+
sig do
|
|
31
|
+
returns(T.nilable(ContextDev::Models::BatchDeleteResponse::KeyMetadata))
|
|
32
|
+
end
|
|
33
|
+
attr_reader :key_metadata
|
|
34
|
+
|
|
35
|
+
sig do
|
|
36
|
+
params(
|
|
37
|
+
key_metadata:
|
|
38
|
+
ContextDev::Models::BatchDeleteResponse::KeyMetadata::OrHash
|
|
39
|
+
).void
|
|
40
|
+
end
|
|
41
|
+
attr_writer :key_metadata
|
|
42
|
+
|
|
43
|
+
sig do
|
|
44
|
+
params(
|
|
45
|
+
id: String,
|
|
46
|
+
deleted: T::Boolean,
|
|
47
|
+
key_metadata:
|
|
48
|
+
ContextDev::Models::BatchDeleteResponse::KeyMetadata::OrHash
|
|
49
|
+
).returns(T.attached_class)
|
|
50
|
+
end
|
|
51
|
+
def self.new(
|
|
52
|
+
# ID of the deleted batch.
|
|
53
|
+
id: nil,
|
|
54
|
+
# Always true on success.
|
|
55
|
+
deleted: nil,
|
|
56
|
+
# Metadata about the API key used for the request. Included in every response
|
|
57
|
+
# whenever a valid API key is provided, even when the response status is not 200.
|
|
58
|
+
key_metadata: nil
|
|
59
|
+
)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
sig do
|
|
63
|
+
override.returns(
|
|
64
|
+
{
|
|
65
|
+
id: String,
|
|
66
|
+
deleted: T::Boolean,
|
|
67
|
+
key_metadata: ContextDev::Models::BatchDeleteResponse::KeyMetadata
|
|
68
|
+
}
|
|
69
|
+
)
|
|
70
|
+
end
|
|
71
|
+
def to_hash
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
class KeyMetadata < ContextDev::Internal::Type::BaseModel
|
|
75
|
+
OrHash =
|
|
76
|
+
T.type_alias do
|
|
77
|
+
T.any(
|
|
78
|
+
ContextDev::Models::BatchDeleteResponse::KeyMetadata,
|
|
79
|
+
ContextDev::Internal::AnyHash
|
|
80
|
+
)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# The number of credits consumed by this request.
|
|
84
|
+
sig { returns(Integer) }
|
|
85
|
+
attr_accessor :credits_consumed
|
|
86
|
+
|
|
87
|
+
# The number of credits remaining for your organization after this request.
|
|
88
|
+
sig { returns(Integer) }
|
|
89
|
+
attr_accessor :credits_remaining
|
|
90
|
+
|
|
91
|
+
# Metadata about the API key used for the request. Included in every response
|
|
92
|
+
# whenever a valid API key is provided, even when the response status is not 200.
|
|
93
|
+
sig do
|
|
94
|
+
params(credits_consumed: Integer, credits_remaining: Integer).returns(
|
|
95
|
+
T.attached_class
|
|
96
|
+
)
|
|
97
|
+
end
|
|
98
|
+
def self.new(
|
|
99
|
+
# The number of credits consumed by this request.
|
|
100
|
+
credits_consumed:,
|
|
101
|
+
# The number of credits remaining for your organization after this request.
|
|
102
|
+
credits_remaining:
|
|
103
|
+
)
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
sig do
|
|
107
|
+
override.returns(
|
|
108
|
+
{ credits_consumed: Integer, credits_remaining: Integer }
|
|
109
|
+
)
|
|
110
|
+
end
|
|
111
|
+
def to_hash
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
@@ -190,6 +190,14 @@ module ContextDev
|
|
|
190
190
|
sig { params(meta: T::Hash[Symbol, T.anything]).void }
|
|
191
191
|
attr_writer :meta
|
|
192
192
|
|
|
193
|
+
# PDF pages of this document recovered by OCR (pdf.ocr=true). Each recovered page
|
|
194
|
+
# bills 1 credit on top of the page base credit; absent when no OCR ran.
|
|
195
|
+
sig { returns(T.nilable(Integer)) }
|
|
196
|
+
attr_reader :ocr_pages
|
|
197
|
+
|
|
198
|
+
sig { params(ocr_pages: Integer).void }
|
|
199
|
+
attr_writer :ocr_pages
|
|
200
|
+
|
|
193
201
|
# A page the batch fetched successfully.
|
|
194
202
|
sig do
|
|
195
203
|
params(
|
|
@@ -202,6 +210,7 @@ module ContextDev
|
|
|
202
210
|
item_id: String,
|
|
203
211
|
markdown: String,
|
|
204
212
|
meta: T::Hash[Symbol, T.anything],
|
|
213
|
+
ocr_pages: Integer,
|
|
205
214
|
status: Symbol
|
|
206
215
|
).returns(T.attached_class)
|
|
207
216
|
end
|
|
@@ -222,6 +231,9 @@ module ContextDev
|
|
|
222
231
|
markdown: nil,
|
|
223
232
|
# Caller-supplied metadata echoed from submission.
|
|
224
233
|
meta: nil,
|
|
234
|
+
# PDF pages of this document recovered by OCR (pdf.ocr=true). Each recovered page
|
|
235
|
+
# bills 1 credit on top of the page base credit; absent when no OCR ran.
|
|
236
|
+
ocr_pages: nil,
|
|
225
237
|
# The page was scraped.
|
|
226
238
|
status: :ok
|
|
227
239
|
)
|
|
@@ -239,7 +251,8 @@ module ContextDev
|
|
|
239
251
|
html: String,
|
|
240
252
|
item_id: String,
|
|
241
253
|
markdown: String,
|
|
242
|
-
meta: T::Hash[Symbol, T.anything]
|
|
254
|
+
meta: T::Hash[Symbol, T.anything],
|
|
255
|
+
ocr_pages: Integer
|
|
243
256
|
}
|
|
244
257
|
)
|
|
245
258
|
end
|
|
@@ -304,11 +304,16 @@ module ContextDev
|
|
|
304
304
|
)
|
|
305
305
|
end
|
|
306
306
|
|
|
307
|
-
# `reserved` minus `refunded` — what the batch has cost so far.
|
|
308
|
-
# `reserved` until the batch settles.
|
|
307
|
+
# `reserved` minus `refunded` plus `ocr_charged` — what the batch has cost so far.
|
|
308
|
+
# Equal to `reserved` until the batch settles.
|
|
309
309
|
sig { returns(Integer) }
|
|
310
310
|
attr_accessor :net
|
|
311
311
|
|
|
312
|
+
# Credits charged for PDF pages recovered by OCR (pdf.ocr=true), 1 per recovered
|
|
313
|
+
# page, on top of `reserved`. Stays 0 until the batch settles.
|
|
314
|
+
sig { returns(Integer) }
|
|
315
|
+
attr_accessor :ocr_charged
|
|
316
|
+
|
|
312
317
|
# Credits returned for pages that did not succeed. Stays 0 until the batch reaches
|
|
313
318
|
# a final status, then settles in one movement.
|
|
314
319
|
sig { returns(Integer) }
|
|
@@ -321,14 +326,20 @@ module ContextDev
|
|
|
321
326
|
|
|
322
327
|
# What this batch has done to your credit balance.
|
|
323
328
|
sig do
|
|
324
|
-
params(
|
|
325
|
-
|
|
326
|
-
|
|
329
|
+
params(
|
|
330
|
+
net: Integer,
|
|
331
|
+
ocr_charged: Integer,
|
|
332
|
+
refunded: Integer,
|
|
333
|
+
reserved: Integer
|
|
334
|
+
).returns(T.attached_class)
|
|
327
335
|
end
|
|
328
336
|
def self.new(
|
|
329
|
-
# `reserved` minus `refunded` — what the batch has cost so far.
|
|
330
|
-
# `reserved` until the batch settles.
|
|
337
|
+
# `reserved` minus `refunded` plus `ocr_charged` — what the batch has cost so far.
|
|
338
|
+
# Equal to `reserved` until the batch settles.
|
|
331
339
|
net:,
|
|
340
|
+
# Credits charged for PDF pages recovered by OCR (pdf.ocr=true), 1 per recovered
|
|
341
|
+
# page, on top of `reserved`. Stays 0 until the batch settles.
|
|
342
|
+
ocr_charged:,
|
|
332
343
|
# Credits returned for pages that did not succeed. Stays 0 until the batch reaches
|
|
333
344
|
# a final status, then settles in one movement.
|
|
334
345
|
refunded:,
|
|
@@ -340,7 +351,12 @@ module ContextDev
|
|
|
340
351
|
|
|
341
352
|
sig do
|
|
342
353
|
override.returns(
|
|
343
|
-
{
|
|
354
|
+
{
|
|
355
|
+
net: Integer,
|
|
356
|
+
ocr_charged: Integer,
|
|
357
|
+
refunded: Integer,
|
|
358
|
+
reserved: Integer
|
|
359
|
+
}
|
|
344
360
|
)
|
|
345
361
|
end
|
|
346
362
|
def to_hash
|