context.dev 1.16.0 → 1.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +19 -0
- data/README.md +3 -3
- data/lib/context_dev/models/ai_extract_product_params.rb +4 -3
- data/lib/context_dev/models/ai_extract_products_params.rb +8 -6
- data/lib/context_dev/models/web_screenshot_params.rb +18 -21
- data/lib/context_dev/models/web_web_crawl_md_params.rb +72 -8
- data/lib/context_dev/models/web_web_crawl_md_response.rb +3 -2
- data/lib/context_dev/models/web_web_scrape_html_params.rb +61 -8
- data/lib/context_dev/models/web_web_scrape_images_params.rb +20 -1
- data/lib/context_dev/models/web_web_scrape_md_params.rb +61 -8
- data/lib/context_dev/models/web_web_scrape_sitemap_params.rb +11 -1
- data/lib/context_dev/resources/ai.rb +1 -1
- data/lib/context_dev/resources/web.rb +46 -16
- data/lib/context_dev/version.rb +1 -1
- data/rbi/context_dev/models/ai_extract_product_params.rbi +6 -4
- data/rbi/context_dev/models/ai_extract_products_params.rbi +12 -8
- data/rbi/context_dev/models/web_screenshot_params.rbi +28 -53
- data/rbi/context_dev/models/web_web_crawl_md_params.rbi +118 -13
- data/rbi/context_dev/models/web_web_crawl_md_response.rbi +4 -2
- data/rbi/context_dev/models/web_web_scrape_html_params.rbi +101 -13
- data/rbi/context_dev/models/web_web_scrape_images_params.rbi +28 -0
- data/rbi/context_dev/models/web_web_scrape_md_params.rbi +101 -13
- data/rbi/context_dev/models/web_web_scrape_sitemap_params.rbi +15 -0
- data/rbi/context_dev/resources/ai.rbi +3 -2
- data/rbi/context_dev/resources/web.rbi +69 -20
- data/sig/context_dev/models/web_screenshot_params.rbs +13 -19
- data/sig/context_dev/models/web_web_crawl_md_params.rbs +53 -6
- data/sig/context_dev/models/web_web_scrape_html_params.rbs +45 -5
- data/sig/context_dev/models/web_web_scrape_images_params.rbs +15 -1
- data/sig/context_dev/models/web_web_scrape_md_params.rbs +46 -6
- data/sig/context_dev/models/web_web_scrape_sitemap_params.rbs +12 -1
- data/sig/context_dev/resources/web.rbs +15 -4
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 5a6ec86b2fdf43b0c3f9b800d8b93558a0c93fdf23fd72fad27a46ba43f2dccc
|
|
4
|
+
data.tar.gz: 8d5533fd69c08516b77b15b9bdcdfafea6cd18f4be1a77d17daa20bad2a6f9b1
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e98fe1516e060eb780f470e8c5ddc29ba4fdea44c4b11ca40d79d2dfacc7009eb11c95360c19dc0f2864857cc2560ce91569d3587cdaacc8009f757b76a293a8
|
|
7
|
+
data.tar.gz: 515ce97f5f873d21cef0665843408f507ad5ee2221ae9308d336617e57b2980d0c8add55c46193f2e10c580819cf09654213d32d6f15558f17c6adc87114ef32
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,24 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 1.18.0 (2026-05-10)
|
|
4
|
+
|
|
5
|
+
Full Changelog: [v1.17.0...v1.18.0](https://github.com/context-dot-dev/context-ruby-sdk/compare/v1.17.0...v1.18.0)
|
|
6
|
+
|
|
7
|
+
### Features
|
|
8
|
+
|
|
9
|
+
* **api:** api update ([b582c05](https://github.com/context-dot-dev/context-ruby-sdk/commit/b582c05376102bb0cb6f8d4d8c9a2cefdef8c1ec))
|
|
10
|
+
* **api:** api update ([4a4e4bb](https://github.com/context-dot-dev/context-ruby-sdk/commit/4a4e4bbc547662de263a307b213dd7eecd03a61d))
|
|
11
|
+
* **api:** manual updates ([ec963bb](https://github.com/context-dot-dev/context-ruby-sdk/commit/ec963bb99ac36d162552c76fb067e87144f21089))
|
|
12
|
+
|
|
13
|
+
## 1.17.0 (2026-05-09)
|
|
14
|
+
|
|
15
|
+
Full Changelog: [v1.16.0...v1.17.0](https://github.com/context-dot-dev/context-ruby-sdk/compare/v1.16.0...v1.17.0)
|
|
16
|
+
|
|
17
|
+
### Features
|
|
18
|
+
|
|
19
|
+
* **api:** api update ([ae638b0](https://github.com/context-dot-dev/context-ruby-sdk/commit/ae638b04d0e08bf93e2cdd1211236d841d5313b1))
|
|
20
|
+
* **api:** api update ([8376344](https://github.com/context-dot-dev/context-ruby-sdk/commit/8376344a7a72a7ff01906bc3d6a5913a4de60bcb))
|
|
21
|
+
|
|
3
22
|
## 1.16.0 (2026-05-07)
|
|
4
23
|
|
|
5
24
|
Full Changelog: [v1.15.0...v1.16.0](https://github.com/context-dot-dev/context-ruby-sdk/compare/v1.15.0...v1.16.0)
|
data/README.md
CHANGED
|
@@ -8,8 +8,8 @@ It is generated with [Stainless](https://www.stainless.com/).
|
|
|
8
8
|
|
|
9
9
|
Use the Context Dev MCP Server to enable AI assistants to interact with this API, allowing them to explore endpoints, make test requests, and use documentation to help integrate this SDK into your application.
|
|
10
10
|
|
|
11
|
-
[](https://cursor.com/en-US/install-mcp?name=context
|
|
12
|
-
[](https://vscode.stainless.com/mcp/%7B%22name%22%3A%22context
|
|
11
|
+
[](https://cursor.com/en-US/install-mcp?name=context-dev-mcp&config=eyJuYW1lIjoiY29udGV4dC1kZXYtbWNwIiwidHJhbnNwb3J0IjoiaHR0cCIsInVybCI6Imh0dHBzOi8vY29udGV4dC1kZXYuc3RsbWNwLmNvbSIsImhlYWRlcnMiOnsieC1jb250ZXh0LWRldi1hcGkta2V5IjoiTXkgQVBJIEtleSJ9fQ)
|
|
12
|
+
[](https://vscode.stainless.com/mcp/%7B%22name%22%3A%22context-dev-mcp%22%2C%22type%22%3A%22http%22%2C%22url%22%3A%22https%3A%2F%2Fcontext-dev.stlmcp.com%22%2C%22headers%22%3A%7B%22x-context-dev-api-key%22%3A%22My%20API%20Key%22%7D%7D)
|
|
13
13
|
|
|
14
14
|
> Note: You may need to set environment variables in your MCP client.
|
|
15
15
|
|
|
@@ -26,7 +26,7 @@ To use this gem, install via Bundler by adding the following to your application
|
|
|
26
26
|
<!-- x-release-please-start-version -->
|
|
27
27
|
|
|
28
28
|
```ruby
|
|
29
|
-
gem "context.dev", "~> 1.
|
|
29
|
+
gem "context.dev", "~> 1.18.0"
|
|
30
30
|
```
|
|
31
31
|
|
|
32
32
|
<!-- x-release-please-end -->
|
|
@@ -22,8 +22,9 @@ module ContextDev
|
|
|
22
22
|
optional :max_age_ms, Integer, api_name: :maxAgeMs
|
|
23
23
|
|
|
24
24
|
# @!attribute timeout_ms
|
|
25
|
-
# Optional timeout in milliseconds for the request.
|
|
26
|
-
#
|
|
25
|
+
# Optional timeout in milliseconds for the request. If the request takes longer
|
|
26
|
+
# than this value, it will be aborted with a 408 status code. Maximum allowed
|
|
27
|
+
# value is 300000ms (5 minutes).
|
|
27
28
|
#
|
|
28
29
|
# @return [Integer, nil]
|
|
29
30
|
optional :timeout_ms, Integer, api_name: :timeoutMS
|
|
@@ -36,7 +37,7 @@ module ContextDev
|
|
|
36
37
|
#
|
|
37
38
|
# @param max_age_ms [Integer] Return a cached result if a prior scrape for the same parameters exists and is y
|
|
38
39
|
#
|
|
39
|
-
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request.
|
|
40
|
+
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
40
41
|
#
|
|
41
42
|
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}]
|
|
42
43
|
end
|
|
@@ -45,8 +45,9 @@ module ContextDev
|
|
|
45
45
|
optional :max_products, Integer, api_name: :maxProducts
|
|
46
46
|
|
|
47
47
|
# @!attribute timeout_ms
|
|
48
|
-
# Optional timeout in milliseconds for the request.
|
|
49
|
-
#
|
|
48
|
+
# Optional timeout in milliseconds for the request. If the request takes longer
|
|
49
|
+
# than this value, it will be aborted with a 408 status code. Maximum allowed
|
|
50
|
+
# value is 300000ms (5 minutes).
|
|
50
51
|
#
|
|
51
52
|
# @return [Integer, nil]
|
|
52
53
|
optional :timeout_ms, Integer, api_name: :timeoutMS
|
|
@@ -61,7 +62,7 @@ module ContextDev
|
|
|
61
62
|
#
|
|
62
63
|
# @param max_products [Integer] Maximum number of products to extract.
|
|
63
64
|
#
|
|
64
|
-
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request.
|
|
65
|
+
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
65
66
|
end
|
|
66
67
|
|
|
67
68
|
class ByDirectURL < ContextDev::Internal::Type::BaseModel
|
|
@@ -87,8 +88,9 @@ module ContextDev
|
|
|
87
88
|
optional :max_products, Integer, api_name: :maxProducts
|
|
88
89
|
|
|
89
90
|
# @!attribute timeout_ms
|
|
90
|
-
# Optional timeout in milliseconds for the request.
|
|
91
|
-
#
|
|
91
|
+
# Optional timeout in milliseconds for the request. If the request takes longer
|
|
92
|
+
# than this value, it will be aborted with a 408 status code. Maximum allowed
|
|
93
|
+
# value is 300000ms (5 minutes).
|
|
92
94
|
#
|
|
93
95
|
# @return [Integer, nil]
|
|
94
96
|
optional :timeout_ms, Integer, api_name: :timeoutMS
|
|
@@ -104,7 +106,7 @@ module ContextDev
|
|
|
104
106
|
#
|
|
105
107
|
# @param max_products [Integer] Maximum number of products to extract.
|
|
106
108
|
#
|
|
107
|
-
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request.
|
|
109
|
+
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
108
110
|
end
|
|
109
111
|
|
|
110
112
|
# @!method self.variants
|
|
@@ -49,13 +49,13 @@ module ContextDev
|
|
|
49
49
|
# @return [Symbol, ContextDev::Models::WebScreenshotParams::Page, nil]
|
|
50
50
|
optional :page, enum: -> { ContextDev::WebScreenshotParams::Page }
|
|
51
51
|
|
|
52
|
-
# @!attribute
|
|
53
|
-
# Optional
|
|
54
|
-
#
|
|
55
|
-
#
|
|
52
|
+
# @!attribute timeout_ms
|
|
53
|
+
# Optional timeout in milliseconds for the request. If the request takes longer
|
|
54
|
+
# than this value, it will be aborted with a 408 status code. Maximum allowed
|
|
55
|
+
# value is 300000ms (5 minutes).
|
|
56
56
|
#
|
|
57
|
-
# @return [
|
|
58
|
-
optional :
|
|
57
|
+
# @return [Integer, nil]
|
|
58
|
+
optional :timeout_ms, Integer
|
|
59
59
|
|
|
60
60
|
# @!attribute viewport
|
|
61
61
|
# Optional browser viewport dimensions for the screenshot. Defaults to 1920x1080.
|
|
@@ -63,7 +63,15 @@ module ContextDev
|
|
|
63
63
|
# @return [ContextDev::Models::WebScreenshotParams::Viewport, nil]
|
|
64
64
|
optional :viewport, -> { ContextDev::WebScreenshotParams::Viewport }
|
|
65
65
|
|
|
66
|
-
# @!
|
|
66
|
+
# @!attribute wait_for_ms
|
|
67
|
+
# Optional browser wait time in milliseconds after initial page load before taking
|
|
68
|
+
# the screenshot. Min: 0. Max: 30000 (30 seconds). Defaults to 3000 ms when
|
|
69
|
+
# omitted.
|
|
70
|
+
#
|
|
71
|
+
# @return [Integer, nil]
|
|
72
|
+
optional :wait_for_ms, Integer
|
|
73
|
+
|
|
74
|
+
# @!method initialize(direct_url: nil, domain: nil, full_screenshot: nil, max_age_ms: nil, page: nil, timeout_ms: nil, viewport: nil, wait_for_ms: nil, request_options: {})
|
|
67
75
|
# Some parameter documentations has been truncated, see
|
|
68
76
|
# {ContextDev::Models::WebScreenshotParams} for more details.
|
|
69
77
|
#
|
|
@@ -77,10 +85,12 @@ module ContextDev
|
|
|
77
85
|
#
|
|
78
86
|
# @param page [Symbol, ContextDev::Models::WebScreenshotParams::Page] Optional parameter to specify which page type to screenshot. If provided, the sy
|
|
79
87
|
#
|
|
80
|
-
# @param
|
|
88
|
+
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
81
89
|
#
|
|
82
90
|
# @param viewport [ContextDev::Models::WebScreenshotParams::Viewport] Optional browser viewport dimensions for the screenshot. Defaults to 1920x1080.
|
|
83
91
|
#
|
|
92
|
+
# @param wait_for_ms [Integer] Optional browser wait time in milliseconds after initial page load before taking
|
|
93
|
+
#
|
|
84
94
|
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}]
|
|
85
95
|
|
|
86
96
|
# Optional parameter to determine screenshot type. If 'true', takes a full page
|
|
@@ -117,19 +127,6 @@ module ContextDev
|
|
|
117
127
|
# @return [Array<Symbol>]
|
|
118
128
|
end
|
|
119
129
|
|
|
120
|
-
# Optional parameter to prioritize screenshot capture. If 'speed', optimizes for
|
|
121
|
-
# faster capture with basic quality. If 'quality', optimizes for higher quality
|
|
122
|
-
# with longer wait times. Defaults to 'quality' if not provided.
|
|
123
|
-
module Prioritize
|
|
124
|
-
extend ContextDev::Internal::Type::Enum
|
|
125
|
-
|
|
126
|
-
SPEED = :speed
|
|
127
|
-
QUALITY = :quality
|
|
128
|
-
|
|
129
|
-
# @!method self.values
|
|
130
|
-
# @return [Array<Symbol>]
|
|
131
|
-
end
|
|
132
|
-
|
|
133
130
|
class Viewport < ContextDev::Internal::Type::BaseModel
|
|
134
131
|
# @!attribute height
|
|
135
132
|
# Viewport height in pixels.
|
|
@@ -60,13 +60,12 @@ module ContextDev
|
|
|
60
60
|
# @return [Integer, nil]
|
|
61
61
|
optional :max_pages, Integer, api_name: :maxPages
|
|
62
62
|
|
|
63
|
-
# @!attribute
|
|
64
|
-
#
|
|
65
|
-
#
|
|
66
|
-
# entirely (not included in results and not counted as failures).
|
|
63
|
+
# @!attribute pdf
|
|
64
|
+
# PDF parsing controls. Use start/end to limit text extraction and OCR to an
|
|
65
|
+
# inclusive 1-based page range.
|
|
67
66
|
#
|
|
68
|
-
# @return [
|
|
69
|
-
optional :
|
|
67
|
+
# @return [ContextDev::Models::WebWebCrawlMdParams::Pdf, nil]
|
|
68
|
+
optional :pdf, -> { ContextDev::WebWebCrawlMdParams::Pdf }
|
|
70
69
|
|
|
71
70
|
# @!attribute shorten_base64_images
|
|
72
71
|
# Truncate base64-encoded image data in the Markdown output
|
|
@@ -74,6 +73,23 @@ module ContextDev
|
|
|
74
73
|
# @return [Boolean, nil]
|
|
75
74
|
optional :shorten_base64_images, ContextDev::Internal::Type::Boolean, api_name: :shortenBase64Images
|
|
76
75
|
|
|
76
|
+
# @!attribute stop_after_ms
|
|
77
|
+
# Soft time budget for the crawl in milliseconds. After each scrape, the crawler
|
|
78
|
+
# checks the elapsed time and, if exceeded, returns the pages collected so far
|
|
79
|
+
# instead of continuing. Min: 10000 (10s). Max: 240000 (4 min). Default: 120000 (2
|
|
80
|
+
# min).
|
|
81
|
+
#
|
|
82
|
+
# @return [Integer, nil]
|
|
83
|
+
optional :stop_after_ms, Integer, api_name: :stopAfterMs
|
|
84
|
+
|
|
85
|
+
# @!attribute timeout_ms
|
|
86
|
+
# Optional timeout in milliseconds for the request. If the request takes longer
|
|
87
|
+
# than this value, it will be aborted with a 408 status code. Maximum allowed
|
|
88
|
+
# value is 300000ms (5 minutes).
|
|
89
|
+
#
|
|
90
|
+
# @return [Integer, nil]
|
|
91
|
+
optional :timeout_ms, Integer, api_name: :timeoutMS
|
|
92
|
+
|
|
77
93
|
# @!attribute url_regex
|
|
78
94
|
# Regex pattern. Only URLs matching this pattern will be followed and scraped.
|
|
79
95
|
#
|
|
@@ -87,7 +103,14 @@ module ContextDev
|
|
|
87
103
|
# @return [Boolean, nil]
|
|
88
104
|
optional :use_main_content_only, ContextDev::Internal::Type::Boolean, api_name: :useMainContentOnly
|
|
89
105
|
|
|
90
|
-
# @!
|
|
106
|
+
# @!attribute wait_for_ms
|
|
107
|
+
# Optional browser wait time in milliseconds after initial page load for each
|
|
108
|
+
# crawled page. Min: 0. Max: 30000 (30 seconds).
|
|
109
|
+
#
|
|
110
|
+
# @return [Integer, nil]
|
|
111
|
+
optional :wait_for_ms, Integer, api_name: :waitForMs
|
|
112
|
+
|
|
113
|
+
# @!method initialize(url:, follow_subdomains: nil, include_frames: nil, include_images: nil, include_links: nil, max_age_ms: nil, max_depth: nil, max_pages: nil, pdf: nil, shorten_base64_images: nil, stop_after_ms: nil, timeout_ms: nil, url_regex: nil, use_main_content_only: nil, wait_for_ms: nil, request_options: {})
|
|
91
114
|
# Some parameter documentations has been truncated, see
|
|
92
115
|
# {ContextDev::Models::WebWebCrawlMdParams} for more details.
|
|
93
116
|
#
|
|
@@ -107,15 +130,56 @@ module ContextDev
|
|
|
107
130
|
#
|
|
108
131
|
# @param max_pages [Integer] Maximum number of pages to crawl. Hard cap: 500.
|
|
109
132
|
#
|
|
110
|
-
# @param
|
|
133
|
+
# @param pdf [ContextDev::Models::WebWebCrawlMdParams::Pdf] PDF parsing controls. Use start/end to limit text extraction and OCR to an inclu
|
|
111
134
|
#
|
|
112
135
|
# @param shorten_base64_images [Boolean] Truncate base64-encoded image data in the Markdown output
|
|
113
136
|
#
|
|
137
|
+
# @param stop_after_ms [Integer] Soft time budget for the crawl in milliseconds. After each scrape, the crawler c
|
|
138
|
+
#
|
|
139
|
+
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
140
|
+
#
|
|
114
141
|
# @param url_regex [String] Regex pattern. Only URLs matching this pattern will be followed and scraped.
|
|
115
142
|
#
|
|
116
143
|
# @param use_main_content_only [Boolean] Extract only the main content, stripping headers, footers, sidebars, and navigat
|
|
117
144
|
#
|
|
145
|
+
# @param wait_for_ms [Integer] Optional browser wait time in milliseconds after initial page load for each craw
|
|
146
|
+
#
|
|
118
147
|
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}]
|
|
148
|
+
|
|
149
|
+
class Pdf < ContextDev::Internal::Type::BaseModel
|
|
150
|
+
# @!attribute end_
|
|
151
|
+
# Last 1-based PDF page to parse. When omitted, parsing ends at the last page.
|
|
152
|
+
# Must be greater than or equal to start when both are provided.
|
|
153
|
+
#
|
|
154
|
+
# @return [Integer, nil]
|
|
155
|
+
optional :end_, Integer, api_name: :end
|
|
156
|
+
|
|
157
|
+
# @!attribute should_parse
|
|
158
|
+
# When true, PDF pages are fetched and parsed. When false, PDF pages are skipped
|
|
159
|
+
# entirely (not included in results and not counted as failures).
|
|
160
|
+
#
|
|
161
|
+
# @return [Boolean, nil]
|
|
162
|
+
optional :should_parse, ContextDev::Internal::Type::Boolean, api_name: :shouldParse
|
|
163
|
+
|
|
164
|
+
# @!attribute start
|
|
165
|
+
# First 1-based PDF page to parse. When omitted, parsing starts at the first page.
|
|
166
|
+
#
|
|
167
|
+
# @return [Integer, nil]
|
|
168
|
+
optional :start, Integer
|
|
169
|
+
|
|
170
|
+
# @!method initialize(end_: nil, should_parse: nil, start: nil)
|
|
171
|
+
# Some parameter documentations has been truncated, see
|
|
172
|
+
# {ContextDev::Models::WebWebCrawlMdParams::Pdf} for more details.
|
|
173
|
+
#
|
|
174
|
+
# PDF parsing controls. Use start/end to limit text extraction and OCR to an
|
|
175
|
+
# inclusive 1-based page range.
|
|
176
|
+
#
|
|
177
|
+
# @param end_ [Integer] Last 1-based PDF page to parse. When omitted, parsing ends at the last page. Mus
|
|
178
|
+
#
|
|
179
|
+
# @param should_parse [Boolean] When true, PDF pages are fetched and parsed. When false, PDF pages are skipped e
|
|
180
|
+
#
|
|
181
|
+
# @param start [Integer] First 1-based PDF page to parse. When omitted, parsing starts at the first page.
|
|
182
|
+
end
|
|
119
183
|
end
|
|
120
184
|
end
|
|
121
185
|
end
|
|
@@ -34,7 +34,8 @@ module ContextDev
|
|
|
34
34
|
required :num_failed, Integer, api_name: :numFailed
|
|
35
35
|
|
|
36
36
|
# @!attribute num_skipped
|
|
37
|
-
# Number of URLs skipped (PDFs when
|
|
37
|
+
# Number of URLs skipped (PDFs when pdf.shouldParse=false, or URLs not matching
|
|
38
|
+
# urlRegex)
|
|
38
39
|
#
|
|
39
40
|
# @return [Integer]
|
|
40
41
|
required :num_skipped, Integer, api_name: :numSkipped
|
|
@@ -59,7 +60,7 @@ module ContextDev
|
|
|
59
60
|
#
|
|
60
61
|
# @param num_failed [Integer] Number of pages that failed to crawl
|
|
61
62
|
#
|
|
62
|
-
# @param num_skipped [Integer] Number of URLs skipped (PDFs when
|
|
63
|
+
# @param num_skipped [Integer] Number of URLs skipped (PDFs when pdf.shouldParse=false, or URLs not matching ur
|
|
63
64
|
#
|
|
64
65
|
# @param num_succeeded [Integer] Number of pages successfully crawled
|
|
65
66
|
#
|
|
@@ -27,15 +27,29 @@ module ContextDev
|
|
|
27
27
|
# @return [Integer, nil]
|
|
28
28
|
optional :max_age_ms, Integer
|
|
29
29
|
|
|
30
|
-
# @!attribute
|
|
31
|
-
#
|
|
32
|
-
#
|
|
33
|
-
# and a 400 WEBSITE_ACCESS_ERROR is returned.
|
|
30
|
+
# @!attribute pdf
|
|
31
|
+
# PDF parsing controls. Use start/end to limit text extraction and OCR to an
|
|
32
|
+
# inclusive 1-based page range.
|
|
34
33
|
#
|
|
35
|
-
# @return [
|
|
36
|
-
optional :
|
|
34
|
+
# @return [ContextDev::Models::WebWebScrapeHTMLParams::Pdf, nil]
|
|
35
|
+
optional :pdf, -> { ContextDev::WebWebScrapeHTMLParams::Pdf }
|
|
36
|
+
|
|
37
|
+
# @!attribute timeout_ms
|
|
38
|
+
# Optional timeout in milliseconds for the request. If the request takes longer
|
|
39
|
+
# than this value, it will be aborted with a 408 status code. Maximum allowed
|
|
40
|
+
# value is 300000ms (5 minutes).
|
|
41
|
+
#
|
|
42
|
+
# @return [Integer, nil]
|
|
43
|
+
optional :timeout_ms, Integer
|
|
44
|
+
|
|
45
|
+
# @!attribute wait_for_ms
|
|
46
|
+
# Optional browser wait time in milliseconds after initial page load. Min: 0. Max:
|
|
47
|
+
# 30000 (30 seconds).
|
|
48
|
+
#
|
|
49
|
+
# @return [Integer, nil]
|
|
50
|
+
optional :wait_for_ms, Integer
|
|
37
51
|
|
|
38
|
-
# @!method initialize(url:, include_frames: nil, max_age_ms: nil,
|
|
52
|
+
# @!method initialize(url:, include_frames: nil, max_age_ms: nil, pdf: nil, timeout_ms: nil, wait_for_ms: nil, request_options: {})
|
|
39
53
|
# Some parameter documentations has been truncated, see
|
|
40
54
|
# {ContextDev::Models::WebWebScrapeHTMLParams} for more details.
|
|
41
55
|
#
|
|
@@ -45,9 +59,48 @@ module ContextDev
|
|
|
45
59
|
#
|
|
46
60
|
# @param max_age_ms [Integer] Return a cached result if a prior scrape for the same parameters exists and is y
|
|
47
61
|
#
|
|
48
|
-
# @param
|
|
62
|
+
# @param pdf [ContextDev::Models::WebWebScrapeHTMLParams::Pdf] PDF parsing controls. Use start/end to limit text extraction and OCR to an inclu
|
|
63
|
+
#
|
|
64
|
+
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
65
|
+
#
|
|
66
|
+
# @param wait_for_ms [Integer] Optional browser wait time in milliseconds after initial page load. Min: 0. Max:
|
|
49
67
|
#
|
|
50
68
|
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}]
|
|
69
|
+
|
|
70
|
+
class Pdf < ContextDev::Internal::Type::BaseModel
|
|
71
|
+
# @!attribute end_
|
|
72
|
+
# Last 1-based PDF page to parse. When omitted, parsing ends at the last page.
|
|
73
|
+
# Must be greater than or equal to start when both are provided.
|
|
74
|
+
#
|
|
75
|
+
# @return [Integer, nil]
|
|
76
|
+
optional :end_, Integer, api_name: :end
|
|
77
|
+
|
|
78
|
+
# @!attribute should_parse
|
|
79
|
+
# When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and
|
|
80
|
+
# a 400 WEBSITE_ACCESS_ERROR is returned.
|
|
81
|
+
#
|
|
82
|
+
# @return [Boolean, nil]
|
|
83
|
+
optional :should_parse, ContextDev::Internal::Type::Boolean, api_name: :shouldParse
|
|
84
|
+
|
|
85
|
+
# @!attribute start
|
|
86
|
+
# First 1-based PDF page to parse. When omitted, parsing starts at the first page.
|
|
87
|
+
#
|
|
88
|
+
# @return [Integer, nil]
|
|
89
|
+
optional :start, Integer
|
|
90
|
+
|
|
91
|
+
# @!method initialize(end_: nil, should_parse: nil, start: nil)
|
|
92
|
+
# Some parameter documentations has been truncated, see
|
|
93
|
+
# {ContextDev::Models::WebWebScrapeHTMLParams::Pdf} for more details.
|
|
94
|
+
#
|
|
95
|
+
# PDF parsing controls. Use start/end to limit text extraction and OCR to an
|
|
96
|
+
# inclusive 1-based page range.
|
|
97
|
+
#
|
|
98
|
+
# @param end_ [Integer] Last 1-based PDF page to parse. When omitted, parsing ends at the last page. Mus
|
|
99
|
+
#
|
|
100
|
+
# @param should_parse [Boolean] When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and
|
|
101
|
+
#
|
|
102
|
+
# @param start [Integer] First 1-based PDF page to parse. When omitted, parsing starts at the first page.
|
|
103
|
+
end
|
|
51
104
|
end
|
|
52
105
|
end
|
|
53
106
|
end
|
|
@@ -27,7 +27,22 @@ module ContextDev
|
|
|
27
27
|
# @return [Integer, nil]
|
|
28
28
|
optional :max_age_ms, Integer
|
|
29
29
|
|
|
30
|
-
# @!
|
|
30
|
+
# @!attribute timeout_ms
|
|
31
|
+
# Optional timeout in milliseconds for the request. If the request takes longer
|
|
32
|
+
# than this value, it will be aborted with a 408 status code. Maximum allowed
|
|
33
|
+
# value is 300000ms (5 minutes).
|
|
34
|
+
#
|
|
35
|
+
# @return [Integer, nil]
|
|
36
|
+
optional :timeout_ms, Integer
|
|
37
|
+
|
|
38
|
+
# @!attribute wait_for_ms
|
|
39
|
+
# Optional browser wait time in milliseconds after initial page load before
|
|
40
|
+
# collecting images. Min: 0. Max: 30000 (30 seconds).
|
|
41
|
+
#
|
|
42
|
+
# @return [Integer, nil]
|
|
43
|
+
optional :wait_for_ms, Integer
|
|
44
|
+
|
|
45
|
+
# @!method initialize(url:, enrichment: nil, max_age_ms: nil, timeout_ms: nil, wait_for_ms: nil, request_options: {})
|
|
31
46
|
# Some parameter documentations has been truncated, see
|
|
32
47
|
# {ContextDev::Models::WebWebScrapeImagesParams} for more details.
|
|
33
48
|
#
|
|
@@ -37,6 +52,10 @@ module ContextDev
|
|
|
37
52
|
#
|
|
38
53
|
# @param max_age_ms [Integer] Reuse a cached result this many milliseconds old or newer. Default: 86400000 (1
|
|
39
54
|
#
|
|
55
|
+
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
56
|
+
#
|
|
57
|
+
# @param wait_for_ms [Integer] Optional browser wait time in milliseconds after initial page load before collec
|
|
58
|
+
#
|
|
40
59
|
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}]
|
|
41
60
|
|
|
42
61
|
class Enrichment < ContextDev::Internal::Type::BaseModel
|
|
@@ -40,13 +40,12 @@ module ContextDev
|
|
|
40
40
|
# @return [Integer, nil]
|
|
41
41
|
optional :max_age_ms, Integer
|
|
42
42
|
|
|
43
|
-
# @!attribute
|
|
44
|
-
#
|
|
45
|
-
#
|
|
46
|
-
# WEBSITE_ACCESS_ERROR is returned.
|
|
43
|
+
# @!attribute pdf
|
|
44
|
+
# PDF parsing controls. Use start/end to limit text extraction and OCR to an
|
|
45
|
+
# inclusive 1-based page range.
|
|
47
46
|
#
|
|
48
|
-
# @return [
|
|
49
|
-
optional :
|
|
47
|
+
# @return [ContextDev::Models::WebWebScrapeMdParams::Pdf, nil]
|
|
48
|
+
optional :pdf, -> { ContextDev::WebWebScrapeMdParams::Pdf }
|
|
50
49
|
|
|
51
50
|
# @!attribute shorten_base64_images
|
|
52
51
|
# Shorten base64-encoded image data in the Markdown output
|
|
@@ -54,6 +53,14 @@ module ContextDev
|
|
|
54
53
|
# @return [Boolean, nil]
|
|
55
54
|
optional :shorten_base64_images, ContextDev::Internal::Type::Boolean
|
|
56
55
|
|
|
56
|
+
# @!attribute timeout_ms
|
|
57
|
+
# Optional timeout in milliseconds for the request. If the request takes longer
|
|
58
|
+
# than this value, it will be aborted with a 408 status code. Maximum allowed
|
|
59
|
+
# value is 300000ms (5 minutes).
|
|
60
|
+
#
|
|
61
|
+
# @return [Integer, nil]
|
|
62
|
+
optional :timeout_ms, Integer
|
|
63
|
+
|
|
57
64
|
# @!attribute use_main_content_only
|
|
58
65
|
# Extract only the main content of the page, excluding headers, footers, sidebars,
|
|
59
66
|
# and navigation
|
|
@@ -61,7 +68,14 @@ module ContextDev
|
|
|
61
68
|
# @return [Boolean, nil]
|
|
62
69
|
optional :use_main_content_only, ContextDev::Internal::Type::Boolean
|
|
63
70
|
|
|
64
|
-
# @!
|
|
71
|
+
# @!attribute wait_for_ms
|
|
72
|
+
# Optional browser wait time in milliseconds after initial page load before
|
|
73
|
+
# converting the page to Markdown. Min: 0. Max: 30000 (30 seconds).
|
|
74
|
+
#
|
|
75
|
+
# @return [Integer, nil]
|
|
76
|
+
optional :wait_for_ms, Integer
|
|
77
|
+
|
|
78
|
+
# @!method initialize(url:, include_frames: nil, include_images: nil, include_links: nil, max_age_ms: nil, pdf: nil, shorten_base64_images: nil, timeout_ms: nil, use_main_content_only: nil, wait_for_ms: nil, request_options: {})
|
|
65
79
|
# Some parameter documentations has been truncated, see
|
|
66
80
|
# {ContextDev::Models::WebWebScrapeMdParams} for more details.
|
|
67
81
|
#
|
|
@@ -75,13 +89,52 @@ module ContextDev
|
|
|
75
89
|
#
|
|
76
90
|
# @param max_age_ms [Integer] Return a cached result if a prior scrape for the same parameters exists and is y
|
|
77
91
|
#
|
|
78
|
-
# @param
|
|
92
|
+
# @param pdf [ContextDev::Models::WebWebScrapeMdParams::Pdf] PDF parsing controls. Use start/end to limit text extraction and OCR to an inclu
|
|
79
93
|
#
|
|
80
94
|
# @param shorten_base64_images [Boolean] Shorten base64-encoded image data in the Markdown output
|
|
81
95
|
#
|
|
96
|
+
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
97
|
+
#
|
|
82
98
|
# @param use_main_content_only [Boolean] Extract only the main content of the page, excluding headers, footers, sidebars,
|
|
83
99
|
#
|
|
100
|
+
# @param wait_for_ms [Integer] Optional browser wait time in milliseconds after initial page load before conver
|
|
101
|
+
#
|
|
84
102
|
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}]
|
|
103
|
+
|
|
104
|
+
class Pdf < ContextDev::Internal::Type::BaseModel
|
|
105
|
+
# @!attribute end_
|
|
106
|
+
# Last 1-based PDF page to parse. When omitted, parsing ends at the last page.
|
|
107
|
+
# Must be greater than or equal to start when both are provided.
|
|
108
|
+
#
|
|
109
|
+
# @return [Integer, nil]
|
|
110
|
+
optional :end_, Integer, api_name: :end
|
|
111
|
+
|
|
112
|
+
# @!attribute should_parse
|
|
113
|
+
# When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and
|
|
114
|
+
# a 400 WEBSITE_ACCESS_ERROR is returned.
|
|
115
|
+
#
|
|
116
|
+
# @return [Boolean, nil]
|
|
117
|
+
optional :should_parse, ContextDev::Internal::Type::Boolean, api_name: :shouldParse
|
|
118
|
+
|
|
119
|
+
# @!attribute start
|
|
120
|
+
# First 1-based PDF page to parse. When omitted, parsing starts at the first page.
|
|
121
|
+
#
|
|
122
|
+
# @return [Integer, nil]
|
|
123
|
+
optional :start, Integer
|
|
124
|
+
|
|
125
|
+
# @!method initialize(end_: nil, should_parse: nil, start: nil)
|
|
126
|
+
# Some parameter documentations has been truncated, see
|
|
127
|
+
# {ContextDev::Models::WebWebScrapeMdParams::Pdf} for more details.
|
|
128
|
+
#
|
|
129
|
+
# PDF parsing controls. Use start/end to limit text extraction and OCR to an
|
|
130
|
+
# inclusive 1-based page range.
|
|
131
|
+
#
|
|
132
|
+
# @param end_ [Integer] Last 1-based PDF page to parse. When omitted, parsing ends at the last page. Mus
|
|
133
|
+
#
|
|
134
|
+
# @param should_parse [Boolean] When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and
|
|
135
|
+
#
|
|
136
|
+
# @param start [Integer] First 1-based PDF page to parse. When omitted, parsing starts at the first page.
|
|
137
|
+
end
|
|
85
138
|
end
|
|
86
139
|
end
|
|
87
140
|
end
|
|
@@ -20,6 +20,14 @@ module ContextDev
|
|
|
20
20
|
# @return [Integer, nil]
|
|
21
21
|
optional :max_links, Integer
|
|
22
22
|
|
|
23
|
+
# @!attribute timeout_ms
|
|
24
|
+
# Optional timeout in milliseconds for the request. If the request takes longer
|
|
25
|
+
# than this value, it will be aborted with a 408 status code. Maximum allowed
|
|
26
|
+
# value is 300000ms (5 minutes).
|
|
27
|
+
#
|
|
28
|
+
# @return [Integer, nil]
|
|
29
|
+
optional :timeout_ms, Integer
|
|
30
|
+
|
|
23
31
|
# @!attribute url_regex
|
|
24
32
|
# Optional RE2-compatible regex pattern. Only URLs matching this pattern are
|
|
25
33
|
# returned and counted against maxLinks.
|
|
@@ -27,7 +35,7 @@ module ContextDev
|
|
|
27
35
|
# @return [String, nil]
|
|
28
36
|
optional :url_regex, String
|
|
29
37
|
|
|
30
|
-
# @!method initialize(domain:, max_links: nil, url_regex: nil, request_options: {})
|
|
38
|
+
# @!method initialize(domain:, max_links: nil, timeout_ms: nil, url_regex: nil, request_options: {})
|
|
31
39
|
# Some parameter documentations has been truncated, see
|
|
32
40
|
# {ContextDev::Models::WebWebScrapeSitemapParams} for more details.
|
|
33
41
|
#
|
|
@@ -35,6 +43,8 @@ module ContextDev
|
|
|
35
43
|
#
|
|
36
44
|
# @param max_links [Integer] Maximum number of links to return from the sitemap crawl. Defaults to 10,000. Mi
|
|
37
45
|
#
|
|
46
|
+
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
47
|
+
#
|
|
38
48
|
# @param url_regex [String] Optional RE2-compatible regex pattern. Only URLs matching this pattern are retur
|
|
39
49
|
#
|
|
40
50
|
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}]
|
|
@@ -48,7 +48,7 @@ module ContextDev
|
|
|
48
48
|
#
|
|
49
49
|
# @param max_age_ms [Integer] Return a cached result if a prior scrape for the same parameters exists and is y
|
|
50
50
|
#
|
|
51
|
-
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request.
|
|
51
|
+
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
52
52
|
#
|
|
53
53
|
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}, nil]
|
|
54
54
|
#
|