webscraping_ai 3.2.1 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +25 -0
- data/LICENSE +21 -0
- data/README.md +110 -85
- data/lib/webscraping_ai/client.rb +130 -0
- data/lib/webscraping_ai/configuration.rb +10 -300
- data/lib/webscraping_ai/errors.rb +44 -0
- data/lib/webscraping_ai/query_encoder.rb +74 -0
- data/lib/webscraping_ai/version.rb +1 -13
- data/lib/webscraping_ai.rb +15 -40
- data/webscraping_ai.gemspec +33 -36
- metadata +27 -74
- data/Gemfile +0 -9
- data/Rakefile +0 -10
- data/docs/AIApi.md +0 -209
- data/docs/Account.md +0 -24
- data/docs/AccountApi.md +0 -76
- data/docs/Error.md +0 -24
- data/docs/HTMLApi.md +0 -109
- data/docs/SelectedHTMLApi.md +0 -209
- data/docs/TextApi.md +0 -109
- data/git_push.sh +0 -57
- data/lib/webscraping_ai/api/account_api.rb +0 -79
- data/lib/webscraping_ai/api/ai_api.rb +0 -295
- data/lib/webscraping_ai/api/html_api.rb +0 -160
- data/lib/webscraping_ai/api/selected_html_api.rb +0 -291
- data/lib/webscraping_ai/api/text_api.rb +0 -160
- data/lib/webscraping_ai/api_client.rb +0 -397
- data/lib/webscraping_ai/api_error.rb +0 -58
- data/lib/webscraping_ai/api_model_base.rb +0 -88
- data/lib/webscraping_ai/models/account.rb +0 -178
- data/lib/webscraping_ai/models/error.rb +0 -178
- data/spec/api/account_api_spec.rb +0 -46
- data/spec/api/ai_api_spec.rb +0 -86
- data/spec/api/html_api_spec.rb +0 -61
- data/spec/api/selected_html_api_spec.rb +0 -86
- data/spec/api/text_api_spec.rb +0 -61
- data/spec/models/account_spec.rb +0 -54
- data/spec/models/error_spec.rb +0 -54
- data/spec/spec_helper.rb +0 -111
data/docs/SelectedHTMLApi.md
DELETED
|
@@ -1,209 +0,0 @@
|
|
|
1
|
-
# WebScrapingAI::SelectedHTMLApi
|
|
2
|
-
|
|
3
|
-
All URIs are relative to *https://api.webscraping.ai*
|
|
4
|
-
|
|
5
|
-
| Method | HTTP request | Description |
|
|
6
|
-
| ------ | ------------ | ----------- |
|
|
7
|
-
| [**get_selected**](SelectedHTMLApi.md#get_selected) | **GET** /selected | HTML of a selected page area by URL and CSS selector |
|
|
8
|
-
| [**get_selected_multiple**](SelectedHTMLApi.md#get_selected_multiple) | **GET** /selected-multiple | HTML of multiple page areas by URL and CSS selectors |
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
## get_selected
|
|
12
|
-
|
|
13
|
-
> String get_selected(url, opts)
|
|
14
|
-
|
|
15
|
-
HTML of a selected page area by URL and CSS selector
|
|
16
|
-
|
|
17
|
-
Returns HTML of a selected page area by URL and CSS selector. Useful if you don't want to do the HTML parsing on your side.
|
|
18
|
-
|
|
19
|
-
### Examples
|
|
20
|
-
|
|
21
|
-
```ruby
|
|
22
|
-
require 'time'
|
|
23
|
-
require 'webscraping_ai'
|
|
24
|
-
# setup authorization
|
|
25
|
-
WebScrapingAI.configure do |config|
|
|
26
|
-
# Configure API key authorization: api_key
|
|
27
|
-
config.api_key['api_key'] = 'YOUR API KEY'
|
|
28
|
-
# Uncomment the following line to set a prefix for the API key, e.g. 'Bearer' (defaults to nil)
|
|
29
|
-
# config.api_key_prefix['api_key'] = 'Bearer'
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
api_instance = WebScrapingAI::SelectedHTMLApi.new
|
|
33
|
-
url = 'https://example.com' # String | URL of the target page.
|
|
34
|
-
opts = {
|
|
35
|
-
selector: 'h1', # String | CSS selector (null by default, returns whole page HTML)
|
|
36
|
-
headers: { key: { key: 'inner_example'}}, # Hash<String, String> | HTTP headers to pass to the target page. Can be specified either via a nested query parameter (...&headers[One]=value1&headers=[Another]=value2) or as a JSON encoded object (...&headers={\"One\": \"value1\", \"Another\": \"value2\"}).
|
|
37
|
-
timeout: 10000, # Integer | Maximum web page retrieval time in ms. Increase it in case of timeout errors (10000 by default, maximum is 30000).
|
|
38
|
-
js: true, # Boolean | Execute on-page JavaScript using a headless browser (true by default).
|
|
39
|
-
js_timeout: 2000, # Integer | Maximum JavaScript rendering time in ms. Increase it in case if you see a loading indicator instead of data on the target page.
|
|
40
|
-
wait_for: 'wait_for_example', # String | CSS selector to wait for before returning the page content. Useful for pages with dynamic content loading. Overrides js_timeout.
|
|
41
|
-
proxy: 'datacenter', # String | Type of proxy. Use `residential` if your site restricts traffic from datacenters, or `stealth` for the most heavily protected sites with advanced anti-bot detection (`datacenter` by default). Residential and stealth proxy requests are more expensive than datacenter, see the pricing page for details.
|
|
42
|
-
country: 'us', # String | Country of the proxy to use (US by default).
|
|
43
|
-
custom_proxy: 'custom_proxy_example', # String | Your own proxy URL to use instead of our built-in proxy pool in \"http://user:password@host:port\" format (<a target=\"_blank\" href=\"https://webscraping.ai/proxies/smartproxy\">Smartproxy</a> for example).
|
|
44
|
-
device: 'desktop', # String | Type of device emulation.
|
|
45
|
-
error_on_404: false, # Boolean | Return error on 404 HTTP status on the target page (false by default).
|
|
46
|
-
error_on_redirect: false, # Boolean | Return error on redirect on the target page (false by default).
|
|
47
|
-
js_script: 'document.querySelector('button').click();', # String | Custom JavaScript code to execute on the target page.
|
|
48
|
-
format: 'json' # String | Format of the response (text by default). \"json\" will return a JSON object with the response, \"text\" will return a plain text/HTML response.
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
begin
|
|
52
|
-
# HTML of a selected page area by URL and CSS selector
|
|
53
|
-
result = api_instance.get_selected(url, opts)
|
|
54
|
-
p result
|
|
55
|
-
rescue WebScrapingAI::ApiError => e
|
|
56
|
-
puts "Error when calling SelectedHTMLApi->get_selected: #{e}"
|
|
57
|
-
end
|
|
58
|
-
```
|
|
59
|
-
|
|
60
|
-
#### Using the get_selected_with_http_info variant
|
|
61
|
-
|
|
62
|
-
This returns an Array which contains the response data, status code and headers.
|
|
63
|
-
|
|
64
|
-
> <Array(String, Integer, Hash)> get_selected_with_http_info(url, opts)
|
|
65
|
-
|
|
66
|
-
```ruby
|
|
67
|
-
begin
|
|
68
|
-
# HTML of a selected page area by URL and CSS selector
|
|
69
|
-
data, status_code, headers = api_instance.get_selected_with_http_info(url, opts)
|
|
70
|
-
p status_code # => 2xx
|
|
71
|
-
p headers # => { ... }
|
|
72
|
-
p data # => String
|
|
73
|
-
rescue WebScrapingAI::ApiError => e
|
|
74
|
-
puts "Error when calling SelectedHTMLApi->get_selected_with_http_info: #{e}"
|
|
75
|
-
end
|
|
76
|
-
```
|
|
77
|
-
|
|
78
|
-
### Parameters
|
|
79
|
-
|
|
80
|
-
| Name | Type | Description | Notes |
|
|
81
|
-
| ---- | ---- | ----------- | ----- |
|
|
82
|
-
| **url** | **String** | URL of the target page. | |
|
|
83
|
-
| **selector** | **String** | CSS selector (null by default, returns whole page HTML) | [optional] |
|
|
84
|
-
| **headers** | [**Hash<String, String>**](String.md) | HTTP headers to pass to the target page. Can be specified either via a nested query parameter (...&headers[One]=value1&headers=[Another]=value2) or as a JSON encoded object (...&headers={\"One\": \"value1\", \"Another\": \"value2\"}). | [optional] |
|
|
85
|
-
| **timeout** | **Integer** | Maximum web page retrieval time in ms. Increase it in case of timeout errors (10000 by default, maximum is 30000). | [optional][default to 10000] |
|
|
86
|
-
| **js** | **Boolean** | Execute on-page JavaScript using a headless browser (true by default). | [optional][default to true] |
|
|
87
|
-
| **js_timeout** | **Integer** | Maximum JavaScript rendering time in ms. Increase it in case if you see a loading indicator instead of data on the target page. | [optional][default to 2000] |
|
|
88
|
-
| **wait_for** | **String** | CSS selector to wait for before returning the page content. Useful for pages with dynamic content loading. Overrides js_timeout. | [optional] |
|
|
89
|
-
| **proxy** | **String** | Type of proxy. Use `residential` if your site restricts traffic from datacenters, or `stealth` for the most heavily protected sites with advanced anti-bot detection (`datacenter` by default). Residential and stealth proxy requests are more expensive than datacenter, see the pricing page for details. | [optional][default to 'datacenter'] |
|
|
90
|
-
| **country** | **String** | Country of the proxy to use (US by default). | [optional][default to 'us'] |
|
|
91
|
-
| **custom_proxy** | **String** | Your own proxy URL to use instead of our built-in proxy pool in \"http://user:password@host:port\" format (<a target=\"_blank\" href=\"https://webscraping.ai/proxies/smartproxy\">Smartproxy</a> for example). | [optional] |
|
|
92
|
-
| **device** | **String** | Type of device emulation. | [optional][default to 'desktop'] |
|
|
93
|
-
| **error_on_404** | **Boolean** | Return error on 404 HTTP status on the target page (false by default). | [optional][default to false] |
|
|
94
|
-
| **error_on_redirect** | **Boolean** | Return error on redirect on the target page (false by default). | [optional][default to false] |
|
|
95
|
-
| **js_script** | **String** | Custom JavaScript code to execute on the target page. | [optional] |
|
|
96
|
-
| **format** | **String** | Format of the response (text by default). \"json\" will return a JSON object with the response, \"text\" will return a plain text/HTML response. | [optional][default to 'json'] |
|
|
97
|
-
|
|
98
|
-
### Return type
|
|
99
|
-
|
|
100
|
-
**String**
|
|
101
|
-
|
|
102
|
-
### Authorization
|
|
103
|
-
|
|
104
|
-
[api_key](../README.md#api_key)
|
|
105
|
-
|
|
106
|
-
### HTTP request headers
|
|
107
|
-
|
|
108
|
-
- **Content-Type**: Not defined
|
|
109
|
-
- **Accept**: application/json, text/html
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
## get_selected_multiple
|
|
113
|
-
|
|
114
|
-
> Array<String> get_selected_multiple(url, opts)
|
|
115
|
-
|
|
116
|
-
HTML of multiple page areas by URL and CSS selectors
|
|
117
|
-
|
|
118
|
-
Returns HTML of multiple page areas by URL and CSS selectors. Useful if you don't want to do the HTML parsing on your side.
|
|
119
|
-
|
|
120
|
-
### Examples
|
|
121
|
-
|
|
122
|
-
```ruby
|
|
123
|
-
require 'time'
|
|
124
|
-
require 'webscraping_ai'
|
|
125
|
-
# setup authorization
|
|
126
|
-
WebScrapingAI.configure do |config|
|
|
127
|
-
# Configure API key authorization: api_key
|
|
128
|
-
config.api_key['api_key'] = 'YOUR API KEY'
|
|
129
|
-
# Uncomment the following line to set a prefix for the API key, e.g. 'Bearer' (defaults to nil)
|
|
130
|
-
# config.api_key_prefix['api_key'] = 'Bearer'
|
|
131
|
-
end
|
|
132
|
-
|
|
133
|
-
api_instance = WebScrapingAI::SelectedHTMLApi.new
|
|
134
|
-
url = 'https://example.com' # String | URL of the target page.
|
|
135
|
-
opts = {
|
|
136
|
-
selectors: ['inner_example'], # Array<String> | Multiple CSS selectors (null by default, returns whole page HTML)
|
|
137
|
-
headers: { key: { key: 'inner_example'}}, # Hash<String, String> | HTTP headers to pass to the target page. Can be specified either via a nested query parameter (...&headers[One]=value1&headers=[Another]=value2) or as a JSON encoded object (...&headers={\"One\": \"value1\", \"Another\": \"value2\"}).
|
|
138
|
-
timeout: 10000, # Integer | Maximum web page retrieval time in ms. Increase it in case of timeout errors (10000 by default, maximum is 30000).
|
|
139
|
-
js: true, # Boolean | Execute on-page JavaScript using a headless browser (true by default).
|
|
140
|
-
js_timeout: 2000, # Integer | Maximum JavaScript rendering time in ms. Increase it in case if you see a loading indicator instead of data on the target page.
|
|
141
|
-
wait_for: 'wait_for_example', # String | CSS selector to wait for before returning the page content. Useful for pages with dynamic content loading. Overrides js_timeout.
|
|
142
|
-
proxy: 'datacenter', # String | Type of proxy. Use `residential` if your site restricts traffic from datacenters, or `stealth` for the most heavily protected sites with advanced anti-bot detection (`datacenter` by default). Residential and stealth proxy requests are more expensive than datacenter, see the pricing page for details.
|
|
143
|
-
country: 'us', # String | Country of the proxy to use (US by default).
|
|
144
|
-
custom_proxy: 'custom_proxy_example', # String | Your own proxy URL to use instead of our built-in proxy pool in \"http://user:password@host:port\" format (<a target=\"_blank\" href=\"https://webscraping.ai/proxies/smartproxy\">Smartproxy</a> for example).
|
|
145
|
-
device: 'desktop', # String | Type of device emulation.
|
|
146
|
-
error_on_404: false, # Boolean | Return error on 404 HTTP status on the target page (false by default).
|
|
147
|
-
error_on_redirect: false, # Boolean | Return error on redirect on the target page (false by default).
|
|
148
|
-
js_script: 'document.querySelector('button').click();' # String | Custom JavaScript code to execute on the target page.
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
begin
|
|
152
|
-
# HTML of multiple page areas by URL and CSS selectors
|
|
153
|
-
result = api_instance.get_selected_multiple(url, opts)
|
|
154
|
-
p result
|
|
155
|
-
rescue WebScrapingAI::ApiError => e
|
|
156
|
-
puts "Error when calling SelectedHTMLApi->get_selected_multiple: #{e}"
|
|
157
|
-
end
|
|
158
|
-
```
|
|
159
|
-
|
|
160
|
-
#### Using the get_selected_multiple_with_http_info variant
|
|
161
|
-
|
|
162
|
-
This returns an Array which contains the response data, status code and headers.
|
|
163
|
-
|
|
164
|
-
> <Array(Array<String>, Integer, Hash)> get_selected_multiple_with_http_info(url, opts)
|
|
165
|
-
|
|
166
|
-
```ruby
|
|
167
|
-
begin
|
|
168
|
-
# HTML of multiple page areas by URL and CSS selectors
|
|
169
|
-
data, status_code, headers = api_instance.get_selected_multiple_with_http_info(url, opts)
|
|
170
|
-
p status_code # => 2xx
|
|
171
|
-
p headers # => { ... }
|
|
172
|
-
p data # => Array<String>
|
|
173
|
-
rescue WebScrapingAI::ApiError => e
|
|
174
|
-
puts "Error when calling SelectedHTMLApi->get_selected_multiple_with_http_info: #{e}"
|
|
175
|
-
end
|
|
176
|
-
```
|
|
177
|
-
|
|
178
|
-
### Parameters
|
|
179
|
-
|
|
180
|
-
| Name | Type | Description | Notes |
|
|
181
|
-
| ---- | ---- | ----------- | ----- |
|
|
182
|
-
| **url** | **String** | URL of the target page. | |
|
|
183
|
-
| **selectors** | [**Array<String>**](String.md) | Multiple CSS selectors (null by default, returns whole page HTML) | [optional] |
|
|
184
|
-
| **headers** | [**Hash<String, String>**](String.md) | HTTP headers to pass to the target page. Can be specified either via a nested query parameter (...&headers[One]=value1&headers=[Another]=value2) or as a JSON encoded object (...&headers={\"One\": \"value1\", \"Another\": \"value2\"}). | [optional] |
|
|
185
|
-
| **timeout** | **Integer** | Maximum web page retrieval time in ms. Increase it in case of timeout errors (10000 by default, maximum is 30000). | [optional][default to 10000] |
|
|
186
|
-
| **js** | **Boolean** | Execute on-page JavaScript using a headless browser (true by default). | [optional][default to true] |
|
|
187
|
-
| **js_timeout** | **Integer** | Maximum JavaScript rendering time in ms. Increase it in case if you see a loading indicator instead of data on the target page. | [optional][default to 2000] |
|
|
188
|
-
| **wait_for** | **String** | CSS selector to wait for before returning the page content. Useful for pages with dynamic content loading. Overrides js_timeout. | [optional] |
|
|
189
|
-
| **proxy** | **String** | Type of proxy. Use `residential` if your site restricts traffic from datacenters, or `stealth` for the most heavily protected sites with advanced anti-bot detection (`datacenter` by default). Residential and stealth proxy requests are more expensive than datacenter, see the pricing page for details. | [optional][default to 'datacenter'] |
|
|
190
|
-
| **country** | **String** | Country of the proxy to use (US by default). | [optional][default to 'us'] |
|
|
191
|
-
| **custom_proxy** | **String** | Your own proxy URL to use instead of our built-in proxy pool in \"http://user:password@host:port\" format (<a target=\"_blank\" href=\"https://webscraping.ai/proxies/smartproxy\">Smartproxy</a> for example). | [optional] |
|
|
192
|
-
| **device** | **String** | Type of device emulation. | [optional][default to 'desktop'] |
|
|
193
|
-
| **error_on_404** | **Boolean** | Return error on 404 HTTP status on the target page (false by default). | [optional][default to false] |
|
|
194
|
-
| **error_on_redirect** | **Boolean** | Return error on redirect on the target page (false by default). | [optional][default to false] |
|
|
195
|
-
| **js_script** | **String** | Custom JavaScript code to execute on the target page. | [optional] |
|
|
196
|
-
|
|
197
|
-
### Return type
|
|
198
|
-
|
|
199
|
-
**Array<String>**
|
|
200
|
-
|
|
201
|
-
### Authorization
|
|
202
|
-
|
|
203
|
-
[api_key](../README.md#api_key)
|
|
204
|
-
|
|
205
|
-
### HTTP request headers
|
|
206
|
-
|
|
207
|
-
- **Content-Type**: Not defined
|
|
208
|
-
- **Accept**: application/json
|
|
209
|
-
|
data/docs/TextApi.md
DELETED
|
@@ -1,109 +0,0 @@
|
|
|
1
|
-
# WebScrapingAI::TextApi
|
|
2
|
-
|
|
3
|
-
All URIs are relative to *https://api.webscraping.ai*
|
|
4
|
-
|
|
5
|
-
| Method | HTTP request | Description |
|
|
6
|
-
| ------ | ------------ | ----------- |
|
|
7
|
-
| [**get_text**](TextApi.md#get_text) | **GET** /text | Page text by URL |
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
## get_text
|
|
11
|
-
|
|
12
|
-
> String get_text(url, opts)
|
|
13
|
-
|
|
14
|
-
Page text by URL
|
|
15
|
-
|
|
16
|
-
Returns the visible text content of a webpage specified by the URL. Can be used to feed data to LLM models. The response can be in plain text, JSON, or XML format based on the text_format parameter. Proxies and Chromium JavaScript rendering are used for page retrieval and processing. Returns JSON on error.
|
|
17
|
-
|
|
18
|
-
### Examples
|
|
19
|
-
|
|
20
|
-
```ruby
|
|
21
|
-
require 'time'
|
|
22
|
-
require 'webscraping_ai'
|
|
23
|
-
# setup authorization
|
|
24
|
-
WebScrapingAI.configure do |config|
|
|
25
|
-
# Configure API key authorization: api_key
|
|
26
|
-
config.api_key['api_key'] = 'YOUR API KEY'
|
|
27
|
-
# Uncomment the following line to set a prefix for the API key, e.g. 'Bearer' (defaults to nil)
|
|
28
|
-
# config.api_key_prefix['api_key'] = 'Bearer'
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
api_instance = WebScrapingAI::TextApi.new
|
|
32
|
-
url = 'https://example.com' # String | URL of the target page.
|
|
33
|
-
opts = {
|
|
34
|
-
text_format: 'plain', # String | Format of the text response (plain by default). \"plain\" will return only the page body text. \"json\" and \"xml\" will return a json/xml with \"title\", \"description\" and \"content\" keys.
|
|
35
|
-
return_links: false, # Boolean | [Works only with text_format=json] Return links from the page body text (false by default). Useful for building web crawlers.
|
|
36
|
-
headers: { key: { key: 'inner_example'}}, # Hash<String, String> | HTTP headers to pass to the target page. Can be specified either via a nested query parameter (...&headers[One]=value1&headers=[Another]=value2) or as a JSON encoded object (...&headers={\"One\": \"value1\", \"Another\": \"value2\"}).
|
|
37
|
-
timeout: 10000, # Integer | Maximum web page retrieval time in ms. Increase it in case of timeout errors (10000 by default, maximum is 30000).
|
|
38
|
-
js: true, # Boolean | Execute on-page JavaScript using a headless browser (true by default).
|
|
39
|
-
js_timeout: 2000, # Integer | Maximum JavaScript rendering time in ms. Increase it in case if you see a loading indicator instead of data on the target page.
|
|
40
|
-
wait_for: 'wait_for_example', # String | CSS selector to wait for before returning the page content. Useful for pages with dynamic content loading. Overrides js_timeout.
|
|
41
|
-
proxy: 'datacenter', # String | Type of proxy. Use `residential` if your site restricts traffic from datacenters, or `stealth` for the most heavily protected sites with advanced anti-bot detection (`datacenter` by default). Residential and stealth proxy requests are more expensive than datacenter, see the pricing page for details.
|
|
42
|
-
country: 'us', # String | Country of the proxy to use (US by default).
|
|
43
|
-
custom_proxy: 'custom_proxy_example', # String | Your own proxy URL to use instead of our built-in proxy pool in \"http://user:password@host:port\" format (<a target=\"_blank\" href=\"https://webscraping.ai/proxies/smartproxy\">Smartproxy</a> for example).
|
|
44
|
-
device: 'desktop', # String | Type of device emulation.
|
|
45
|
-
error_on_404: false, # Boolean | Return error on 404 HTTP status on the target page (false by default).
|
|
46
|
-
error_on_redirect: false, # Boolean | Return error on redirect on the target page (false by default).
|
|
47
|
-
js_script: 'document.querySelector('button').click();' # String | Custom JavaScript code to execute on the target page.
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
begin
|
|
51
|
-
# Page text by URL
|
|
52
|
-
result = api_instance.get_text(url, opts)
|
|
53
|
-
p result
|
|
54
|
-
rescue WebScrapingAI::ApiError => e
|
|
55
|
-
puts "Error when calling TextApi->get_text: #{e}"
|
|
56
|
-
end
|
|
57
|
-
```
|
|
58
|
-
|
|
59
|
-
#### Using the get_text_with_http_info variant
|
|
60
|
-
|
|
61
|
-
This returns an Array which contains the response data, status code and headers.
|
|
62
|
-
|
|
63
|
-
> <Array(String, Integer, Hash)> get_text_with_http_info(url, opts)
|
|
64
|
-
|
|
65
|
-
```ruby
|
|
66
|
-
begin
|
|
67
|
-
# Page text by URL
|
|
68
|
-
data, status_code, headers = api_instance.get_text_with_http_info(url, opts)
|
|
69
|
-
p status_code # => 2xx
|
|
70
|
-
p headers # => { ... }
|
|
71
|
-
p data # => String
|
|
72
|
-
rescue WebScrapingAI::ApiError => e
|
|
73
|
-
puts "Error when calling TextApi->get_text_with_http_info: #{e}"
|
|
74
|
-
end
|
|
75
|
-
```
|
|
76
|
-
|
|
77
|
-
### Parameters
|
|
78
|
-
|
|
79
|
-
| Name | Type | Description | Notes |
|
|
80
|
-
| ---- | ---- | ----------- | ----- |
|
|
81
|
-
| **url** | **String** | URL of the target page. | |
|
|
82
|
-
| **text_format** | **String** | Format of the text response (plain by default). \"plain\" will return only the page body text. \"json\" and \"xml\" will return a json/xml with \"title\", \"description\" and \"content\" keys. | [optional][default to 'plain'] |
|
|
83
|
-
| **return_links** | **Boolean** | [Works only with text_format=json] Return links from the page body text (false by default). Useful for building web crawlers. | [optional][default to false] |
|
|
84
|
-
| **headers** | [**Hash<String, String>**](String.md) | HTTP headers to pass to the target page. Can be specified either via a nested query parameter (...&headers[One]=value1&headers=[Another]=value2) or as a JSON encoded object (...&headers={\"One\": \"value1\", \"Another\": \"value2\"}). | [optional] |
|
|
85
|
-
| **timeout** | **Integer** | Maximum web page retrieval time in ms. Increase it in case of timeout errors (10000 by default, maximum is 30000). | [optional][default to 10000] |
|
|
86
|
-
| **js** | **Boolean** | Execute on-page JavaScript using a headless browser (true by default). | [optional][default to true] |
|
|
87
|
-
| **js_timeout** | **Integer** | Maximum JavaScript rendering time in ms. Increase it in case if you see a loading indicator instead of data on the target page. | [optional][default to 2000] |
|
|
88
|
-
| **wait_for** | **String** | CSS selector to wait for before returning the page content. Useful for pages with dynamic content loading. Overrides js_timeout. | [optional] |
|
|
89
|
-
| **proxy** | **String** | Type of proxy. Use `residential` if your site restricts traffic from datacenters, or `stealth` for the most heavily protected sites with advanced anti-bot detection (`datacenter` by default). Residential and stealth proxy requests are more expensive than datacenter, see the pricing page for details. | [optional][default to 'datacenter'] |
|
|
90
|
-
| **country** | **String** | Country of the proxy to use (US by default). | [optional][default to 'us'] |
|
|
91
|
-
| **custom_proxy** | **String** | Your own proxy URL to use instead of our built-in proxy pool in \"http://user:password@host:port\" format (<a target=\"_blank\" href=\"https://webscraping.ai/proxies/smartproxy\">Smartproxy</a> for example). | [optional] |
|
|
92
|
-
| **device** | **String** | Type of device emulation. | [optional][default to 'desktop'] |
|
|
93
|
-
| **error_on_404** | **Boolean** | Return error on 404 HTTP status on the target page (false by default). | [optional][default to false] |
|
|
94
|
-
| **error_on_redirect** | **Boolean** | Return error on redirect on the target page (false by default). | [optional][default to false] |
|
|
95
|
-
| **js_script** | **String** | Custom JavaScript code to execute on the target page. | [optional] |
|
|
96
|
-
|
|
97
|
-
### Return type
|
|
98
|
-
|
|
99
|
-
**String**
|
|
100
|
-
|
|
101
|
-
### Authorization
|
|
102
|
-
|
|
103
|
-
[api_key](../README.md#api_key)
|
|
104
|
-
|
|
105
|
-
### HTTP request headers
|
|
106
|
-
|
|
107
|
-
- **Content-Type**: Not defined
|
|
108
|
-
- **Accept**: application/json, text/html, text/xml
|
|
109
|
-
|
data/git_push.sh
DELETED
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
#!/bin/sh
|
|
2
|
-
# ref: https://help.github.com/articles/adding-an-existing-project-to-github-using-the-command-line/
|
|
3
|
-
#
|
|
4
|
-
# Usage example: /bin/sh ./git_push.sh wing328 openapi-petstore-perl "minor update" "gitlab.com"
|
|
5
|
-
|
|
6
|
-
git_user_id=$1
|
|
7
|
-
git_repo_id=$2
|
|
8
|
-
release_note=$3
|
|
9
|
-
git_host=$4
|
|
10
|
-
|
|
11
|
-
if [ "$git_host" = "" ]; then
|
|
12
|
-
git_host="github.com"
|
|
13
|
-
echo "[INFO] No command line input provided. Set \$git_host to $git_host"
|
|
14
|
-
fi
|
|
15
|
-
|
|
16
|
-
if [ "$git_user_id" = "" ]; then
|
|
17
|
-
git_user_id="webscraping-ai"
|
|
18
|
-
echo "[INFO] No command line input provided. Set \$git_user_id to $git_user_id"
|
|
19
|
-
fi
|
|
20
|
-
|
|
21
|
-
if [ "$git_repo_id" = "" ]; then
|
|
22
|
-
git_repo_id="webscraping-ai-ruby"
|
|
23
|
-
echo "[INFO] No command line input provided. Set \$git_repo_id to $git_repo_id"
|
|
24
|
-
fi
|
|
25
|
-
|
|
26
|
-
if [ "$release_note" = "" ]; then
|
|
27
|
-
release_note="Minor update"
|
|
28
|
-
echo "[INFO] No command line input provided. Set \$release_note to $release_note"
|
|
29
|
-
fi
|
|
30
|
-
|
|
31
|
-
# Initialize the local directory as a Git repository
|
|
32
|
-
git init
|
|
33
|
-
|
|
34
|
-
# Adds the files in the local repository and stages them for commit.
|
|
35
|
-
git add .
|
|
36
|
-
|
|
37
|
-
# Commits the tracked changes and prepares them to be pushed to a remote repository.
|
|
38
|
-
git commit -m "$release_note"
|
|
39
|
-
|
|
40
|
-
# Sets the new remote
|
|
41
|
-
git_remote=$(git remote)
|
|
42
|
-
if [ "$git_remote" = "" ]; then # git remote not defined
|
|
43
|
-
|
|
44
|
-
if [ "$GIT_TOKEN" = "" ]; then
|
|
45
|
-
echo "[INFO] \$GIT_TOKEN (environment variable) is not set. Using the git credential in your environment."
|
|
46
|
-
git remote add origin https://${git_host}/${git_user_id}/${git_repo_id}.git
|
|
47
|
-
else
|
|
48
|
-
git remote add origin https://${git_user_id}:"${GIT_TOKEN}"@${git_host}/${git_user_id}/${git_repo_id}.git
|
|
49
|
-
fi
|
|
50
|
-
|
|
51
|
-
fi
|
|
52
|
-
|
|
53
|
-
git pull origin master
|
|
54
|
-
|
|
55
|
-
# Pushes (Forces) the changes in the local repository up to the remote repository
|
|
56
|
-
echo "Git pushing to https://${git_host}/${git_user_id}/${git_repo_id}.git"
|
|
57
|
-
git push origin master 2>&1 | grep -v 'To https'
|
|
@@ -1,79 +0,0 @@
|
|
|
1
|
-
=begin
|
|
2
|
-
#WebScraping.AI
|
|
3
|
-
|
|
4
|
-
#WebScraping.AI scraping API provides LLM-powered tools with Chromium JavaScript rendering, rotating proxies, and built-in HTML parsing.
|
|
5
|
-
|
|
6
|
-
The version of the OpenAPI document: 3.2.1
|
|
7
|
-
Contact: support@webscraping.ai
|
|
8
|
-
Generated by: https://openapi-generator.tech
|
|
9
|
-
Generator version: 7.22.0
|
|
10
|
-
|
|
11
|
-
=end
|
|
12
|
-
|
|
13
|
-
require 'cgi'
|
|
14
|
-
|
|
15
|
-
module WebScrapingAI
|
|
16
|
-
class AccountApi
|
|
17
|
-
attr_accessor :api_client
|
|
18
|
-
|
|
19
|
-
def initialize(api_client = ApiClient.default)
|
|
20
|
-
@api_client = api_client
|
|
21
|
-
end
|
|
22
|
-
# Information about your account calls quota
|
|
23
|
-
# Returns information about your account, including the remaining API credits quota, the next billing cycle start time, and the remaining concurrent requests. The response is in JSON format.
|
|
24
|
-
# @param [Hash] opts the optional parameters
|
|
25
|
-
# @return [Account]
|
|
26
|
-
def account(opts = {})
|
|
27
|
-
data, _status_code, _headers = account_with_http_info(opts)
|
|
28
|
-
data
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
# Information about your account calls quota
|
|
32
|
-
# Returns information about your account, including the remaining API credits quota, the next billing cycle start time, and the remaining concurrent requests. The response is in JSON format.
|
|
33
|
-
# @param [Hash] opts the optional parameters
|
|
34
|
-
# @return [Array<(Account, Integer, Hash)>] Account data, response status code and response headers
|
|
35
|
-
def account_with_http_info(opts = {})
|
|
36
|
-
if @api_client.config.debugging
|
|
37
|
-
@api_client.config.logger.debug 'Calling API: AccountApi.account ...'
|
|
38
|
-
end
|
|
39
|
-
# resource path
|
|
40
|
-
local_var_path = '/account'
|
|
41
|
-
|
|
42
|
-
# query parameters
|
|
43
|
-
query_params = opts[:query_params] || {}
|
|
44
|
-
|
|
45
|
-
# header parameters
|
|
46
|
-
header_params = opts[:header_params] || {}
|
|
47
|
-
# HTTP header 'Accept' (if needed)
|
|
48
|
-
header_params['Accept'] = @api_client.select_header_accept(['application/json']) unless header_params['Accept']
|
|
49
|
-
|
|
50
|
-
# form parameters
|
|
51
|
-
form_params = opts[:form_params] || {}
|
|
52
|
-
|
|
53
|
-
# http body (model)
|
|
54
|
-
post_body = opts[:debug_body]
|
|
55
|
-
|
|
56
|
-
# return_type
|
|
57
|
-
return_type = opts[:debug_return_type] || 'Account'
|
|
58
|
-
|
|
59
|
-
# auth_names
|
|
60
|
-
auth_names = opts[:debug_auth_names] || ['api_key']
|
|
61
|
-
|
|
62
|
-
new_options = opts.merge(
|
|
63
|
-
:operation => :"AccountApi.account",
|
|
64
|
-
:header_params => header_params,
|
|
65
|
-
:query_params => query_params,
|
|
66
|
-
:form_params => form_params,
|
|
67
|
-
:body => post_body,
|
|
68
|
-
:auth_names => auth_names,
|
|
69
|
-
:return_type => return_type
|
|
70
|
-
)
|
|
71
|
-
|
|
72
|
-
data, status_code, headers = @api_client.call_api(:GET, local_var_path, new_options)
|
|
73
|
-
if @api_client.config.debugging
|
|
74
|
-
@api_client.config.logger.debug "API called: AccountApi#account\nData: #{data.inspect}\nStatus code: #{status_code}\nHeaders: #{headers}"
|
|
75
|
-
end
|
|
76
|
-
return data, status_code, headers
|
|
77
|
-
end
|
|
78
|
-
end
|
|
79
|
-
end
|