context.dev 1.15.0 → 1.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +17 -0
- data/README.md +1 -1
- data/lib/context_dev/models/ai_extract_product_params.rb +4 -3
- data/lib/context_dev/models/ai_extract_products_params.rb +8 -6
- data/lib/context_dev/models/web_screenshot_params.rb +55 -19
- data/lib/context_dev/models/web_screenshot_response.rb +17 -1
- data/lib/context_dev/models/web_web_crawl_md_params.rb +20 -1
- data/lib/context_dev/models/web_web_scrape_html_params.rb +20 -1
- data/lib/context_dev/models/web_web_scrape_images_params.rb +20 -1
- data/lib/context_dev/models/web_web_scrape_md_params.rb +20 -1
- data/lib/context_dev/models/web_web_scrape_sitemap_params.rb +11 -1
- data/lib/context_dev/resources/ai.rb +1 -1
- data/lib/context_dev/resources/web.rb +50 -12
- data/lib/context_dev/version.rb +1 -1
- data/rbi/context_dev/models/ai_extract_product_params.rbi +6 -4
- data/rbi/context_dev/models/ai_extract_products_params.rbi +12 -8
- data/rbi/context_dev/models/web_screenshot_params.rbi +85 -42
- data/rbi/context_dev/models/web_screenshot_response.rbi +25 -3
- data/rbi/context_dev/models/web_web_crawl_md_params.rbi +28 -0
- data/rbi/context_dev/models/web_web_scrape_html_params.rbi +28 -0
- data/rbi/context_dev/models/web_web_scrape_images_params.rbi +28 -0
- data/rbi/context_dev/models/web_web_scrape_md_params.rbi +28 -0
- data/rbi/context_dev/models/web_web_scrape_sitemap_params.rbi +15 -0
- data/rbi/context_dev/resources/ai.rbi +3 -2
- data/rbi/context_dev/resources/web.rbi +59 -5
- data/sig/context_dev/models/web_screenshot_params.rbs +39 -13
- data/sig/context_dev/models/web_screenshot_response.rbs +17 -3
- data/sig/context_dev/models/web_web_crawl_md_params.rbs +15 -1
- data/sig/context_dev/models/web_web_scrape_html_params.rbs +15 -1
- data/sig/context_dev/models/web_web_scrape_images_params.rbs +15 -1
- data/sig/context_dev/models/web_web_scrape_md_params.rbs +15 -1
- data/sig/context_dev/models/web_web_scrape_sitemap_params.rbs +12 -1
- data/sig/context_dev/resources/web.rbs +13 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e47becb29d02dd213aee4b6b49429685c9d74c35f37d49c026b8bbabc87be191
|
|
4
|
+
data.tar.gz: 2e76f0c796bf46d9ec0a1f764de47841eabf2cb290c71b279a9740b574393d98
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 859d0da8ad2c2e9fd7d0bdb4b783b9e8c75f4323d3a599b42d28ad495342ff1187f6a502449b1a325090f3ccab3f76c1d855e25c5aaaee1e7e98d690e9ac44e4
|
|
7
|
+
data.tar.gz: 5b30272a7da7dbfeb75475e067f86bb404313ee842aa86b973817ccf359dc07c53fe06890b2e2dc34b689da07ec3b12a2b77a1f871eeaa0c549196f1afbc873d
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,22 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 1.17.0 (2026-05-09)
|
|
4
|
+
|
|
5
|
+
Full Changelog: [v1.16.0...v1.17.0](https://github.com/context-dot-dev/context-ruby-sdk/compare/v1.16.0...v1.17.0)
|
|
6
|
+
|
|
7
|
+
### Features
|
|
8
|
+
|
|
9
|
+
* **api:** api update ([ae638b0](https://github.com/context-dot-dev/context-ruby-sdk/commit/ae638b04d0e08bf93e2cdd1211236d841d5313b1))
|
|
10
|
+
* **api:** api update ([8376344](https://github.com/context-dot-dev/context-ruby-sdk/commit/8376344a7a72a7ff01906bc3d6a5913a4de60bcb))
|
|
11
|
+
|
|
12
|
+
## 1.16.0 (2026-05-07)
|
|
13
|
+
|
|
14
|
+
Full Changelog: [v1.15.0...v1.16.0](https://github.com/context-dot-dev/context-ruby-sdk/compare/v1.15.0...v1.16.0)
|
|
15
|
+
|
|
16
|
+
### Features
|
|
17
|
+
|
|
18
|
+
* **api:** api update ([f0b1f3a](https://github.com/context-dot-dev/context-ruby-sdk/commit/f0b1f3afd885c44cb464393b57d944fa27a6c0b5))
|
|
19
|
+
|
|
3
20
|
## 1.15.0 (2026-05-06)
|
|
4
21
|
|
|
5
22
|
Full Changelog: [v1.14.0...v1.15.0](https://github.com/context-dot-dev/context-ruby-sdk/compare/v1.14.0...v1.15.0)
|
data/README.md
CHANGED
|
@@ -22,8 +22,9 @@ module ContextDev
|
|
|
22
22
|
optional :max_age_ms, Integer, api_name: :maxAgeMs
|
|
23
23
|
|
|
24
24
|
# @!attribute timeout_ms
|
|
25
|
-
# Optional timeout in milliseconds for the request.
|
|
26
|
-
#
|
|
25
|
+
# Optional timeout in milliseconds for the request. If the request takes longer
|
|
26
|
+
# than this value, it will be aborted with a 408 status code. Maximum allowed
|
|
27
|
+
# value is 300000ms (5 minutes).
|
|
27
28
|
#
|
|
28
29
|
# @return [Integer, nil]
|
|
29
30
|
optional :timeout_ms, Integer, api_name: :timeoutMS
|
|
@@ -36,7 +37,7 @@ module ContextDev
|
|
|
36
37
|
#
|
|
37
38
|
# @param max_age_ms [Integer] Return a cached result if a prior scrape for the same parameters exists and is y
|
|
38
39
|
#
|
|
39
|
-
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request.
|
|
40
|
+
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
40
41
|
#
|
|
41
42
|
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}]
|
|
42
43
|
end
|
|
@@ -45,8 +45,9 @@ module ContextDev
|
|
|
45
45
|
optional :max_products, Integer, api_name: :maxProducts
|
|
46
46
|
|
|
47
47
|
# @!attribute timeout_ms
|
|
48
|
-
# Optional timeout in milliseconds for the request.
|
|
49
|
-
#
|
|
48
|
+
# Optional timeout in milliseconds for the request. If the request takes longer
|
|
49
|
+
# than this value, it will be aborted with a 408 status code. Maximum allowed
|
|
50
|
+
# value is 300000ms (5 minutes).
|
|
50
51
|
#
|
|
51
52
|
# @return [Integer, nil]
|
|
52
53
|
optional :timeout_ms, Integer, api_name: :timeoutMS
|
|
@@ -61,7 +62,7 @@ module ContextDev
|
|
|
61
62
|
#
|
|
62
63
|
# @param max_products [Integer] Maximum number of products to extract.
|
|
63
64
|
#
|
|
64
|
-
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request.
|
|
65
|
+
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
65
66
|
end
|
|
66
67
|
|
|
67
68
|
class ByDirectURL < ContextDev::Internal::Type::BaseModel
|
|
@@ -87,8 +88,9 @@ module ContextDev
|
|
|
87
88
|
optional :max_products, Integer, api_name: :maxProducts
|
|
88
89
|
|
|
89
90
|
# @!attribute timeout_ms
|
|
90
|
-
# Optional timeout in milliseconds for the request.
|
|
91
|
-
#
|
|
91
|
+
# Optional timeout in milliseconds for the request. If the request takes longer
|
|
92
|
+
# than this value, it will be aborted with a 408 status code. Maximum allowed
|
|
93
|
+
# value is 300000ms (5 minutes).
|
|
92
94
|
#
|
|
93
95
|
# @return [Integer, nil]
|
|
94
96
|
optional :timeout_ms, Integer, api_name: :timeoutMS
|
|
@@ -104,7 +106,7 @@ module ContextDev
|
|
|
104
106
|
#
|
|
105
107
|
# @param max_products [Integer] Maximum number of products to extract.
|
|
106
108
|
#
|
|
107
|
-
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request.
|
|
109
|
+
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
108
110
|
end
|
|
109
111
|
|
|
110
112
|
# @!method self.variants
|
|
@@ -31,6 +31,14 @@ module ContextDev
|
|
|
31
31
|
# @return [Symbol, ContextDev::Models::WebScreenshotParams::FullScreenshot, nil]
|
|
32
32
|
optional :full_screenshot, enum: -> { ContextDev::WebScreenshotParams::FullScreenshot }
|
|
33
33
|
|
|
34
|
+
# @!attribute max_age_ms
|
|
35
|
+
# Return a cached screenshot if a prior screenshot for the same parameters exists
|
|
36
|
+
# and is younger than this many milliseconds. Defaults to 1 day (86400000 ms) when
|
|
37
|
+
# omitted. Max is 30 days (2592000000 ms). Set to 0 to always capture fresh.
|
|
38
|
+
#
|
|
39
|
+
# @return [Integer, nil]
|
|
40
|
+
optional :max_age_ms, Integer
|
|
41
|
+
|
|
34
42
|
# @!attribute page
|
|
35
43
|
# Optional parameter to specify which page type to screenshot. If provided, the
|
|
36
44
|
# system will scrape the domain's links and use heuristics to find the most
|
|
@@ -41,15 +49,29 @@ module ContextDev
|
|
|
41
49
|
# @return [Symbol, ContextDev::Models::WebScreenshotParams::Page, nil]
|
|
42
50
|
optional :page, enum: -> { ContextDev::WebScreenshotParams::Page }
|
|
43
51
|
|
|
44
|
-
# @!attribute
|
|
45
|
-
# Optional
|
|
46
|
-
#
|
|
47
|
-
#
|
|
52
|
+
# @!attribute timeout_ms
|
|
53
|
+
# Optional timeout in milliseconds for the request. If the request takes longer
|
|
54
|
+
# than this value, it will be aborted with a 408 status code. Maximum allowed
|
|
55
|
+
# value is 300000ms (5 minutes).
|
|
56
|
+
#
|
|
57
|
+
# @return [Integer, nil]
|
|
58
|
+
optional :timeout_ms, Integer
|
|
59
|
+
|
|
60
|
+
# @!attribute viewport
|
|
61
|
+
# Optional browser viewport dimensions for the screenshot. Defaults to 1920x1080.
|
|
48
62
|
#
|
|
49
|
-
# @return [
|
|
50
|
-
optional :
|
|
63
|
+
# @return [ContextDev::Models::WebScreenshotParams::Viewport, nil]
|
|
64
|
+
optional :viewport, -> { ContextDev::WebScreenshotParams::Viewport }
|
|
51
65
|
|
|
52
|
-
# @!
|
|
66
|
+
# @!attribute wait_for_ms
|
|
67
|
+
# Optional browser wait time in milliseconds after initial page load before taking
|
|
68
|
+
# the screenshot. Min: 0. Max: 30000 (30 seconds). Defaults to 3000 ms when
|
|
69
|
+
# omitted.
|
|
70
|
+
#
|
|
71
|
+
# @return [Integer, nil]
|
|
72
|
+
optional :wait_for_ms, Integer
|
|
73
|
+
|
|
74
|
+
# @!method initialize(direct_url: nil, domain: nil, full_screenshot: nil, max_age_ms: nil, page: nil, timeout_ms: nil, viewport: nil, wait_for_ms: nil, request_options: {})
|
|
53
75
|
# Some parameter documentations has been truncated, see
|
|
54
76
|
# {ContextDev::Models::WebScreenshotParams} for more details.
|
|
55
77
|
#
|
|
@@ -59,9 +81,15 @@ module ContextDev
|
|
|
59
81
|
#
|
|
60
82
|
# @param full_screenshot [Symbol, ContextDev::Models::WebScreenshotParams::FullScreenshot] Optional parameter to determine screenshot type. If 'true', takes a full page sc
|
|
61
83
|
#
|
|
84
|
+
# @param max_age_ms [Integer] Return a cached screenshot if a prior screenshot for the same parameters exists
|
|
85
|
+
#
|
|
62
86
|
# @param page [Symbol, ContextDev::Models::WebScreenshotParams::Page] Optional parameter to specify which page type to screenshot. If provided, the sy
|
|
63
87
|
#
|
|
64
|
-
# @param
|
|
88
|
+
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
89
|
+
#
|
|
90
|
+
# @param viewport [ContextDev::Models::WebScreenshotParams::Viewport] Optional browser viewport dimensions for the screenshot. Defaults to 1920x1080.
|
|
91
|
+
#
|
|
92
|
+
# @param wait_for_ms [Integer] Optional browser wait time in milliseconds after initial page load before taking
|
|
65
93
|
#
|
|
66
94
|
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}]
|
|
67
95
|
|
|
@@ -99,17 +127,25 @@ module ContextDev
|
|
|
99
127
|
# @return [Array<Symbol>]
|
|
100
128
|
end
|
|
101
129
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
#
|
|
112
|
-
# @return [
|
|
130
|
+
class Viewport < ContextDev::Internal::Type::BaseModel
|
|
131
|
+
# @!attribute height
|
|
132
|
+
# Viewport height in pixels.
|
|
133
|
+
#
|
|
134
|
+
# @return [Integer, nil]
|
|
135
|
+
optional :height, Integer
|
|
136
|
+
|
|
137
|
+
# @!attribute width
|
|
138
|
+
# Viewport width in pixels.
|
|
139
|
+
#
|
|
140
|
+
# @return [Integer, nil]
|
|
141
|
+
optional :width, Integer
|
|
142
|
+
|
|
143
|
+
# @!method initialize(height: nil, width: nil)
|
|
144
|
+
# Optional browser viewport dimensions for the screenshot. Defaults to 1920x1080.
|
|
145
|
+
#
|
|
146
|
+
# @param height [Integer] Viewport height in pixels.
|
|
147
|
+
#
|
|
148
|
+
# @param width [Integer] Viewport width in pixels.
|
|
113
149
|
end
|
|
114
150
|
end
|
|
115
151
|
end
|
|
@@ -16,6 +16,12 @@ module ContextDev
|
|
|
16
16
|
# @return [String, nil]
|
|
17
17
|
optional :domain, String
|
|
18
18
|
|
|
19
|
+
# @!attribute height
|
|
20
|
+
# Height in pixels of the returned screenshot image
|
|
21
|
+
#
|
|
22
|
+
# @return [Integer, nil]
|
|
23
|
+
optional :height, Integer
|
|
24
|
+
|
|
19
25
|
# @!attribute screenshot
|
|
20
26
|
# Public URL of the uploaded screenshot image
|
|
21
27
|
#
|
|
@@ -36,16 +42,26 @@ module ContextDev
|
|
|
36
42
|
# @return [String, nil]
|
|
37
43
|
optional :status, String
|
|
38
44
|
|
|
39
|
-
# @!
|
|
45
|
+
# @!attribute width
|
|
46
|
+
# Width in pixels of the returned screenshot image
|
|
47
|
+
#
|
|
48
|
+
# @return [Integer, nil]
|
|
49
|
+
optional :width, Integer
|
|
50
|
+
|
|
51
|
+
# @!method initialize(code: nil, domain: nil, height: nil, screenshot: nil, screenshot_type: nil, status: nil, width: nil)
|
|
40
52
|
# @param code [Integer] HTTP status code
|
|
41
53
|
#
|
|
42
54
|
# @param domain [String] The normalized domain that was processed
|
|
43
55
|
#
|
|
56
|
+
# @param height [Integer] Height in pixels of the returned screenshot image
|
|
57
|
+
#
|
|
44
58
|
# @param screenshot [String] Public URL of the uploaded screenshot image
|
|
45
59
|
#
|
|
46
60
|
# @param screenshot_type [Symbol, ContextDev::Models::WebScreenshotResponse::ScreenshotType] Type of screenshot that was captured
|
|
47
61
|
#
|
|
48
62
|
# @param status [String] Status of the response, e.g., 'ok'
|
|
63
|
+
#
|
|
64
|
+
# @param width [Integer] Width in pixels of the returned screenshot image
|
|
49
65
|
|
|
50
66
|
# Type of screenshot that was captured
|
|
51
67
|
#
|
|
@@ -74,6 +74,14 @@ module ContextDev
|
|
|
74
74
|
# @return [Boolean, nil]
|
|
75
75
|
optional :shorten_base64_images, ContextDev::Internal::Type::Boolean, api_name: :shortenBase64Images
|
|
76
76
|
|
|
77
|
+
# @!attribute timeout_ms
|
|
78
|
+
# Optional timeout in milliseconds for the request. If the request takes longer
|
|
79
|
+
# than this value, it will be aborted with a 408 status code. Maximum allowed
|
|
80
|
+
# value is 300000ms (5 minutes).
|
|
81
|
+
#
|
|
82
|
+
# @return [Integer, nil]
|
|
83
|
+
optional :timeout_ms, Integer, api_name: :timeoutMS
|
|
84
|
+
|
|
77
85
|
# @!attribute url_regex
|
|
78
86
|
# Regex pattern. Only URLs matching this pattern will be followed and scraped.
|
|
79
87
|
#
|
|
@@ -87,7 +95,14 @@ module ContextDev
|
|
|
87
95
|
# @return [Boolean, nil]
|
|
88
96
|
optional :use_main_content_only, ContextDev::Internal::Type::Boolean, api_name: :useMainContentOnly
|
|
89
97
|
|
|
90
|
-
# @!
|
|
98
|
+
# @!attribute wait_for_ms
|
|
99
|
+
# Optional browser wait time in milliseconds after initial page load for each
|
|
100
|
+
# crawled page. Min: 0. Max: 30000 (30 seconds).
|
|
101
|
+
#
|
|
102
|
+
# @return [Integer, nil]
|
|
103
|
+
optional :wait_for_ms, Integer, api_name: :waitForMs
|
|
104
|
+
|
|
105
|
+
# @!method initialize(url:, follow_subdomains: nil, include_frames: nil, include_images: nil, include_links: nil, max_age_ms: nil, max_depth: nil, max_pages: nil, parse_pdf: nil, shorten_base64_images: nil, timeout_ms: nil, url_regex: nil, use_main_content_only: nil, wait_for_ms: nil, request_options: {})
|
|
91
106
|
# Some parameter documentations has been truncated, see
|
|
92
107
|
# {ContextDev::Models::WebWebCrawlMdParams} for more details.
|
|
93
108
|
#
|
|
@@ -111,10 +126,14 @@ module ContextDev
|
|
|
111
126
|
#
|
|
112
127
|
# @param shorten_base64_images [Boolean] Truncate base64-encoded image data in the Markdown output
|
|
113
128
|
#
|
|
129
|
+
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
130
|
+
#
|
|
114
131
|
# @param url_regex [String] Regex pattern. Only URLs matching this pattern will be followed and scraped.
|
|
115
132
|
#
|
|
116
133
|
# @param use_main_content_only [Boolean] Extract only the main content, stripping headers, footers, sidebars, and navigat
|
|
117
134
|
#
|
|
135
|
+
# @param wait_for_ms [Integer] Optional browser wait time in milliseconds after initial page load for each craw
|
|
136
|
+
#
|
|
118
137
|
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}]
|
|
119
138
|
end
|
|
120
139
|
end
|
|
@@ -35,7 +35,22 @@ module ContextDev
|
|
|
35
35
|
# @return [Boolean, nil]
|
|
36
36
|
optional :parse_pdf, ContextDev::Internal::Type::Boolean
|
|
37
37
|
|
|
38
|
-
# @!
|
|
38
|
+
# @!attribute timeout_ms
|
|
39
|
+
# Optional timeout in milliseconds for the request. If the request takes longer
|
|
40
|
+
# than this value, it will be aborted with a 408 status code. Maximum allowed
|
|
41
|
+
# value is 300000ms (5 minutes).
|
|
42
|
+
#
|
|
43
|
+
# @return [Integer, nil]
|
|
44
|
+
optional :timeout_ms, Integer
|
|
45
|
+
|
|
46
|
+
# @!attribute wait_for_ms
|
|
47
|
+
# Optional browser wait time in milliseconds after initial page load. Min: 0. Max:
|
|
48
|
+
# 30000 (30 seconds).
|
|
49
|
+
#
|
|
50
|
+
# @return [Integer, nil]
|
|
51
|
+
optional :wait_for_ms, Integer
|
|
52
|
+
|
|
53
|
+
# @!method initialize(url:, include_frames: nil, max_age_ms: nil, parse_pdf: nil, timeout_ms: nil, wait_for_ms: nil, request_options: {})
|
|
39
54
|
# Some parameter documentations has been truncated, see
|
|
40
55
|
# {ContextDev::Models::WebWebScrapeHTMLParams} for more details.
|
|
41
56
|
#
|
|
@@ -47,6 +62,10 @@ module ContextDev
|
|
|
47
62
|
#
|
|
48
63
|
# @param parse_pdf [Boolean] When true (default), PDF URLs are fetched and their text layer is extracted and
|
|
49
64
|
#
|
|
65
|
+
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
66
|
+
#
|
|
67
|
+
# @param wait_for_ms [Integer] Optional browser wait time in milliseconds after initial page load. Min: 0. Max:
|
|
68
|
+
#
|
|
50
69
|
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}]
|
|
51
70
|
end
|
|
52
71
|
end
|
|
@@ -27,7 +27,22 @@ module ContextDev
|
|
|
27
27
|
# @return [Integer, nil]
|
|
28
28
|
optional :max_age_ms, Integer
|
|
29
29
|
|
|
30
|
-
# @!
|
|
30
|
+
# @!attribute timeout_ms
|
|
31
|
+
# Optional timeout in milliseconds for the request. If the request takes longer
|
|
32
|
+
# than this value, it will be aborted with a 408 status code. Maximum allowed
|
|
33
|
+
# value is 300000ms (5 minutes).
|
|
34
|
+
#
|
|
35
|
+
# @return [Integer, nil]
|
|
36
|
+
optional :timeout_ms, Integer
|
|
37
|
+
|
|
38
|
+
# @!attribute wait_for_ms
|
|
39
|
+
# Optional browser wait time in milliseconds after initial page load before
|
|
40
|
+
# collecting images. Min: 0. Max: 30000 (30 seconds).
|
|
41
|
+
#
|
|
42
|
+
# @return [Integer, nil]
|
|
43
|
+
optional :wait_for_ms, Integer
|
|
44
|
+
|
|
45
|
+
# @!method initialize(url:, enrichment: nil, max_age_ms: nil, timeout_ms: nil, wait_for_ms: nil, request_options: {})
|
|
31
46
|
# Some parameter documentations has been truncated, see
|
|
32
47
|
# {ContextDev::Models::WebWebScrapeImagesParams} for more details.
|
|
33
48
|
#
|
|
@@ -37,6 +52,10 @@ module ContextDev
|
|
|
37
52
|
#
|
|
38
53
|
# @param max_age_ms [Integer] Reuse a cached result this many milliseconds old or newer. Default: 86400000 (1
|
|
39
54
|
#
|
|
55
|
+
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
56
|
+
#
|
|
57
|
+
# @param wait_for_ms [Integer] Optional browser wait time in milliseconds after initial page load before collec
|
|
58
|
+
#
|
|
40
59
|
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}]
|
|
41
60
|
|
|
42
61
|
class Enrichment < ContextDev::Internal::Type::BaseModel
|
|
@@ -54,6 +54,14 @@ module ContextDev
|
|
|
54
54
|
# @return [Boolean, nil]
|
|
55
55
|
optional :shorten_base64_images, ContextDev::Internal::Type::Boolean
|
|
56
56
|
|
|
57
|
+
# @!attribute timeout_ms
|
|
58
|
+
# Optional timeout in milliseconds for the request. If the request takes longer
|
|
59
|
+
# than this value, it will be aborted with a 408 status code. Maximum allowed
|
|
60
|
+
# value is 300000ms (5 minutes).
|
|
61
|
+
#
|
|
62
|
+
# @return [Integer, nil]
|
|
63
|
+
optional :timeout_ms, Integer
|
|
64
|
+
|
|
57
65
|
# @!attribute use_main_content_only
|
|
58
66
|
# Extract only the main content of the page, excluding headers, footers, sidebars,
|
|
59
67
|
# and navigation
|
|
@@ -61,7 +69,14 @@ module ContextDev
|
|
|
61
69
|
# @return [Boolean, nil]
|
|
62
70
|
optional :use_main_content_only, ContextDev::Internal::Type::Boolean
|
|
63
71
|
|
|
64
|
-
# @!
|
|
72
|
+
# @!attribute wait_for_ms
|
|
73
|
+
# Optional browser wait time in milliseconds after initial page load before
|
|
74
|
+
# converting the page to Markdown. Min: 0. Max: 30000 (30 seconds).
|
|
75
|
+
#
|
|
76
|
+
# @return [Integer, nil]
|
|
77
|
+
optional :wait_for_ms, Integer
|
|
78
|
+
|
|
79
|
+
# @!method initialize(url:, include_frames: nil, include_images: nil, include_links: nil, max_age_ms: nil, parse_pdf: nil, shorten_base64_images: nil, timeout_ms: nil, use_main_content_only: nil, wait_for_ms: nil, request_options: {})
|
|
65
80
|
# Some parameter documentations has been truncated, see
|
|
66
81
|
# {ContextDev::Models::WebWebScrapeMdParams} for more details.
|
|
67
82
|
#
|
|
@@ -79,8 +94,12 @@ module ContextDev
|
|
|
79
94
|
#
|
|
80
95
|
# @param shorten_base64_images [Boolean] Shorten base64-encoded image data in the Markdown output
|
|
81
96
|
#
|
|
97
|
+
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
98
|
+
#
|
|
82
99
|
# @param use_main_content_only [Boolean] Extract only the main content of the page, excluding headers, footers, sidebars,
|
|
83
100
|
#
|
|
101
|
+
# @param wait_for_ms [Integer] Optional browser wait time in milliseconds after initial page load before conver
|
|
102
|
+
#
|
|
84
103
|
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}]
|
|
85
104
|
end
|
|
86
105
|
end
|
|
@@ -20,6 +20,14 @@ module ContextDev
|
|
|
20
20
|
# @return [Integer, nil]
|
|
21
21
|
optional :max_links, Integer
|
|
22
22
|
|
|
23
|
+
# @!attribute timeout_ms
|
|
24
|
+
# Optional timeout in milliseconds for the request. If the request takes longer
|
|
25
|
+
# than this value, it will be aborted with a 408 status code. Maximum allowed
|
|
26
|
+
# value is 300000ms (5 minutes).
|
|
27
|
+
#
|
|
28
|
+
# @return [Integer, nil]
|
|
29
|
+
optional :timeout_ms, Integer
|
|
30
|
+
|
|
23
31
|
# @!attribute url_regex
|
|
24
32
|
# Optional RE2-compatible regex pattern. Only URLs matching this pattern are
|
|
25
33
|
# returned and counted against maxLinks.
|
|
@@ -27,7 +35,7 @@ module ContextDev
|
|
|
27
35
|
# @return [String, nil]
|
|
28
36
|
optional :url_regex, String
|
|
29
37
|
|
|
30
|
-
# @!method initialize(domain:, max_links: nil, url_regex: nil, request_options: {})
|
|
38
|
+
# @!method initialize(domain:, max_links: nil, timeout_ms: nil, url_regex: nil, request_options: {})
|
|
31
39
|
# Some parameter documentations has been truncated, see
|
|
32
40
|
# {ContextDev::Models::WebWebScrapeSitemapParams} for more details.
|
|
33
41
|
#
|
|
@@ -35,6 +43,8 @@ module ContextDev
|
|
|
35
43
|
#
|
|
36
44
|
# @param max_links [Integer] Maximum number of links to return from the sitemap crawl. Defaults to 10,000. Mi
|
|
37
45
|
#
|
|
46
|
+
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
47
|
+
#
|
|
38
48
|
# @param url_regex [String] Optional RE2-compatible regex pattern. Only URLs matching this pattern are retur
|
|
39
49
|
#
|
|
40
50
|
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}]
|
|
@@ -48,7 +48,7 @@ module ContextDev
|
|
|
48
48
|
#
|
|
49
49
|
# @param max_age_ms [Integer] Return a cached result if a prior scrape for the same parameters exists and is y
|
|
50
50
|
#
|
|
51
|
-
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request.
|
|
51
|
+
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
52
52
|
#
|
|
53
53
|
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}, nil]
|
|
54
54
|
#
|
|
@@ -70,7 +70,7 @@ module ContextDev
|
|
|
70
70
|
#
|
|
71
71
|
# Capture a screenshot of a website.
|
|
72
72
|
#
|
|
73
|
-
# @overload screenshot(direct_url: nil, domain: nil, full_screenshot: nil, page: nil,
|
|
73
|
+
# @overload screenshot(direct_url: nil, domain: nil, full_screenshot: nil, max_age_ms: nil, page: nil, timeout_ms: nil, viewport: nil, wait_for_ms: nil, request_options: {})
|
|
74
74
|
#
|
|
75
75
|
# @param direct_url [String] A specific URL to screenshot directly, bypassing domain resolution (e.g., 'https
|
|
76
76
|
#
|
|
@@ -78,9 +78,15 @@ module ContextDev
|
|
|
78
78
|
#
|
|
79
79
|
# @param full_screenshot [Symbol, ContextDev::Models::WebScreenshotParams::FullScreenshot] Optional parameter to determine screenshot type. If 'true', takes a full page sc
|
|
80
80
|
#
|
|
81
|
+
# @param max_age_ms [Integer] Return a cached screenshot if a prior screenshot for the same parameters exists
|
|
82
|
+
#
|
|
81
83
|
# @param page [Symbol, ContextDev::Models::WebScreenshotParams::Page] Optional parameter to specify which page type to screenshot. If provided, the sy
|
|
82
84
|
#
|
|
83
|
-
# @param
|
|
85
|
+
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
86
|
+
#
|
|
87
|
+
# @param viewport [ContextDev::Models::WebScreenshotParams::Viewport] Optional browser viewport dimensions for the screenshot. Defaults to 1920x1080.
|
|
88
|
+
#
|
|
89
|
+
# @param wait_for_ms [Integer] Optional browser wait time in milliseconds after initial page load before taking
|
|
84
90
|
#
|
|
85
91
|
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}, nil]
|
|
86
92
|
#
|
|
@@ -93,7 +99,13 @@ module ContextDev
|
|
|
93
99
|
@client.request(
|
|
94
100
|
method: :get,
|
|
95
101
|
path: "web/screenshot",
|
|
96
|
-
query: query.transform_keys(
|
|
102
|
+
query: query.transform_keys(
|
|
103
|
+
direct_url: "directUrl",
|
|
104
|
+
full_screenshot: "fullScreenshot",
|
|
105
|
+
max_age_ms: "maxAgeMs",
|
|
106
|
+
timeout_ms: "timeoutMS",
|
|
107
|
+
wait_for_ms: "waitForMs"
|
|
108
|
+
),
|
|
97
109
|
model: ContextDev::Models::WebScreenshotResponse,
|
|
98
110
|
options: options
|
|
99
111
|
)
|
|
@@ -105,7 +117,7 @@ module ContextDev
|
|
|
105
117
|
# Performs a crawl starting from a given URL, extracts page content as Markdown,
|
|
106
118
|
# and returns results for all crawled pages.
|
|
107
119
|
#
|
|
108
|
-
# @overload web_crawl_md(url:, follow_subdomains: nil, include_frames: nil, include_images: nil, include_links: nil, max_age_ms: nil, max_depth: nil, max_pages: nil, parse_pdf: nil, shorten_base64_images: nil, url_regex: nil, use_main_content_only: nil, request_options: {})
|
|
120
|
+
# @overload web_crawl_md(url:, follow_subdomains: nil, include_frames: nil, include_images: nil, include_links: nil, max_age_ms: nil, max_depth: nil, max_pages: nil, parse_pdf: nil, shorten_base64_images: nil, timeout_ms: nil, url_regex: nil, use_main_content_only: nil, wait_for_ms: nil, request_options: {})
|
|
109
121
|
#
|
|
110
122
|
# @param url [String] The starting URL for the crawl (must include http:// or https:// protocol)
|
|
111
123
|
#
|
|
@@ -127,10 +139,14 @@ module ContextDev
|
|
|
127
139
|
#
|
|
128
140
|
# @param shorten_base64_images [Boolean] Truncate base64-encoded image data in the Markdown output
|
|
129
141
|
#
|
|
142
|
+
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
143
|
+
#
|
|
130
144
|
# @param url_regex [String] Regex pattern. Only URLs matching this pattern will be followed and scraped.
|
|
131
145
|
#
|
|
132
146
|
# @param use_main_content_only [Boolean] Extract only the main content, stripping headers, footers, sidebars, and navigat
|
|
133
147
|
#
|
|
148
|
+
# @param wait_for_ms [Integer] Optional browser wait time in milliseconds after initial page load for each craw
|
|
149
|
+
#
|
|
134
150
|
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}, nil]
|
|
135
151
|
#
|
|
136
152
|
# @return [ContextDev::Models::WebWebCrawlMdResponse]
|
|
@@ -152,7 +168,7 @@ module ContextDev
|
|
|
152
168
|
#
|
|
153
169
|
# Scrapes the given URL and returns the raw HTML content of the page.
|
|
154
170
|
#
|
|
155
|
-
# @overload web_scrape_html(url:, include_frames: nil, max_age_ms: nil, parse_pdf: nil, request_options: {})
|
|
171
|
+
# @overload web_scrape_html(url:, include_frames: nil, max_age_ms: nil, parse_pdf: nil, timeout_ms: nil, wait_for_ms: nil, request_options: {})
|
|
156
172
|
#
|
|
157
173
|
# @param url [String] Full URL to scrape (must include http:// or https:// protocol)
|
|
158
174
|
#
|
|
@@ -162,6 +178,10 @@ module ContextDev
|
|
|
162
178
|
#
|
|
163
179
|
# @param parse_pdf [Boolean] When true (default), PDF URLs are fetched and their text layer is extracted and
|
|
164
180
|
#
|
|
181
|
+
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
182
|
+
#
|
|
183
|
+
# @param wait_for_ms [Integer] Optional browser wait time in milliseconds after initial page load. Min: 0. Max:
|
|
184
|
+
#
|
|
165
185
|
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}, nil]
|
|
166
186
|
#
|
|
167
187
|
# @return [ContextDev::Models::WebWebScrapeHTMLResponse]
|
|
@@ -176,7 +196,9 @@ module ContextDev
|
|
|
176
196
|
query: query.transform_keys(
|
|
177
197
|
include_frames: "includeFrames",
|
|
178
198
|
max_age_ms: "maxAgeMs",
|
|
179
|
-
parse_pdf: "parsePDF"
|
|
199
|
+
parse_pdf: "parsePDF",
|
|
200
|
+
timeout_ms: "timeoutMS",
|
|
201
|
+
wait_for_ms: "waitForMs"
|
|
180
202
|
),
|
|
181
203
|
model: ContextDev::Models::WebWebScrapeHTMLResponse,
|
|
182
204
|
options: options
|
|
@@ -191,7 +213,7 @@ module ContextDev
|
|
|
191
213
|
# embeds. The base request costs 1 credit; enrichment costs 1 credit per returned
|
|
192
214
|
# image.
|
|
193
215
|
#
|
|
194
|
-
# @overload web_scrape_images(url:, enrichment: nil, max_age_ms: nil, request_options: {})
|
|
216
|
+
# @overload web_scrape_images(url:, enrichment: nil, max_age_ms: nil, timeout_ms: nil, wait_for_ms: nil, request_options: {})
|
|
195
217
|
#
|
|
196
218
|
# @param url [String] Page URL to inspect. Must include http:// or https://.
|
|
197
219
|
#
|
|
@@ -199,6 +221,10 @@ module ContextDev
|
|
|
199
221
|
#
|
|
200
222
|
# @param max_age_ms [Integer] Reuse a cached result this many milliseconds old or newer. Default: 86400000 (1
|
|
201
223
|
#
|
|
224
|
+
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
225
|
+
#
|
|
226
|
+
# @param wait_for_ms [Integer] Optional browser wait time in milliseconds after initial page load before collec
|
|
227
|
+
#
|
|
202
228
|
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}, nil]
|
|
203
229
|
#
|
|
204
230
|
# @return [ContextDev::Models::WebWebScrapeImagesResponse]
|
|
@@ -210,7 +236,11 @@ module ContextDev
|
|
|
210
236
|
@client.request(
|
|
211
237
|
method: :get,
|
|
212
238
|
path: "web/scrape/images",
|
|
213
|
-
query: query.transform_keys(
|
|
239
|
+
query: query.transform_keys(
|
|
240
|
+
max_age_ms: "maxAgeMs",
|
|
241
|
+
timeout_ms: "timeoutMS",
|
|
242
|
+
wait_for_ms: "waitForMs"
|
|
243
|
+
),
|
|
214
244
|
model: ContextDev::Models::WebWebScrapeImagesResponse,
|
|
215
245
|
options: options
|
|
216
246
|
)
|
|
@@ -221,7 +251,7 @@ module ContextDev
|
|
|
221
251
|
#
|
|
222
252
|
# Scrapes the given URL into LLM usable Markdown.
|
|
223
253
|
#
|
|
224
|
-
# @overload web_scrape_md(url:, include_frames: nil, include_images: nil, include_links: nil, max_age_ms: nil, parse_pdf: nil, shorten_base64_images: nil, use_main_content_only: nil, request_options: {})
|
|
254
|
+
# @overload web_scrape_md(url:, include_frames: nil, include_images: nil, include_links: nil, max_age_ms: nil, parse_pdf: nil, shorten_base64_images: nil, timeout_ms: nil, use_main_content_only: nil, wait_for_ms: nil, request_options: {})
|
|
225
255
|
#
|
|
226
256
|
# @param url [String] Full URL to scrape into LLM usable Markdown (must include http:// or https:// pr
|
|
227
257
|
#
|
|
@@ -237,8 +267,12 @@ module ContextDev
|
|
|
237
267
|
#
|
|
238
268
|
# @param shorten_base64_images [Boolean] Shorten base64-encoded image data in the Markdown output
|
|
239
269
|
#
|
|
270
|
+
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
271
|
+
#
|
|
240
272
|
# @param use_main_content_only [Boolean] Extract only the main content of the page, excluding headers, footers, sidebars,
|
|
241
273
|
#
|
|
274
|
+
# @param wait_for_ms [Integer] Optional browser wait time in milliseconds after initial page load before conver
|
|
275
|
+
#
|
|
242
276
|
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}, nil]
|
|
243
277
|
#
|
|
244
278
|
# @return [ContextDev::Models::WebWebScrapeMdResponse]
|
|
@@ -257,7 +291,9 @@ module ContextDev
|
|
|
257
291
|
max_age_ms: "maxAgeMs",
|
|
258
292
|
parse_pdf: "parsePDF",
|
|
259
293
|
shorten_base64_images: "shortenBase64Images",
|
|
260
|
-
|
|
294
|
+
timeout_ms: "timeoutMS",
|
|
295
|
+
use_main_content_only: "useMainContentOnly",
|
|
296
|
+
wait_for_ms: "waitForMs"
|
|
261
297
|
),
|
|
262
298
|
model: ContextDev::Models::WebWebScrapeMdResponse,
|
|
263
299
|
options: options
|
|
@@ -269,12 +305,14 @@ module ContextDev
|
|
|
269
305
|
#
|
|
270
306
|
# Crawl an entire website's sitemap and return all discovered page URLs.
|
|
271
307
|
#
|
|
272
|
-
# @overload web_scrape_sitemap(domain:, max_links: nil, url_regex: nil, request_options: {})
|
|
308
|
+
# @overload web_scrape_sitemap(domain:, max_links: nil, timeout_ms: nil, url_regex: nil, request_options: {})
|
|
273
309
|
#
|
|
274
310
|
# @param domain [String] Domain to build a sitemap for
|
|
275
311
|
#
|
|
276
312
|
# @param max_links [Integer] Maximum number of links to return from the sitemap crawl. Defaults to 10,000. Mi
|
|
277
313
|
#
|
|
314
|
+
# @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
|
|
315
|
+
#
|
|
278
316
|
# @param url_regex [String] Optional RE2-compatible regex pattern. Only URLs matching this pattern are retur
|
|
279
317
|
#
|
|
280
318
|
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}, nil]
|
|
@@ -288,7 +326,7 @@ module ContextDev
|
|
|
288
326
|
@client.request(
|
|
289
327
|
method: :get,
|
|
290
328
|
path: "web/scrape/sitemap",
|
|
291
|
-
query: query.transform_keys(max_links: "maxLinks", url_regex: "urlRegex"),
|
|
329
|
+
query: query.transform_keys(max_links: "maxLinks", timeout_ms: "timeoutMS", url_regex: "urlRegex"),
|
|
292
330
|
model: ContextDev::Models::WebWebScrapeSitemapResponse,
|
|
293
331
|
options: options
|
|
294
332
|
)
|
data/lib/context_dev/version.rb
CHANGED