webscraping_ai 3.1.3 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b9d5c09646212d1d2b3127a8ab58b7db3c400b725e696e2da3a6049d193d00b1
-  data.tar.gz: f0ef33b178203e092eb99c8f94a9a8b5acab23e09913ac01f09fdb1a36e08a60
+  metadata.gz: ea239c0ee1260cacb701ce616aef022672f3aa35ca5ec44e6db5348eee1e9fa4
+  data.tar.gz: f2a9af9da881dc8c24396847cbfa724a91f1b12da5b2eeab38b492913383d64b
 SHA512:
-  metadata.gz: 7432ef958f2d51ced742d7c7b80e9662456e990e10eed9bbf3a95519253c8788746c9aa5c0cd3c8136c52adb41fac1b1de53b3abfa6daac9a6ca15419b2b780b
-  data.tar.gz: 8cf21004408e4dfa207d4f709ac81de2b2a2851f6c8b86ef2cdf5a28b7f88f3a826ba1ca8b87fb9c60a9e365521a99ffac1cbf8ced043abb3822c010a7357112
+  metadata.gz: 9d9d8624aa0e2159e6009e2ae3719b2cec9dce3a3d51883441159a8ac9ace76fa801c043bbf19950e001519d06226cc102fcdef746ec427648af1b380d963e84
+  data.tar.gz: 6afde37248d490b226cc41ffb41b972d4d6b427878e36ff0a267ea9e6959b080b2309f4e651151cf762fe2819ab52fe712ff5ae6becb2be1c7f5f1df788c5cd4

data/README.md

@@ -2,12 +2,13 @@
 
 WebScrapingAI - the Ruby gem for the WebScraping.AI
 
-WebScraping.AI scraping API provides GPT-powered tools with Chromium JavaScript rendering, rotating proxies, and built-in HTML parsing.
+WebScraping.AI scraping API provides LLM-powered tools with Chromium JavaScript rendering, rotating proxies, and built-in HTML parsing.
 
 This SDK is automatically generated by the [OpenAPI Generator](https://openapi-generator.tech) project:
 
-- API version: 3.1.3
-- Package version: 3.1.3
+- API version: 3.2.0
+- Package version: 3.2.0
+- Generator version: 7.11.0
 - Build package: org.openapitools.codegen.languages.RubyClientCodegen
 For more information, please visit [https://webscraping.ai](https://webscraping.ai)
 
@@ -24,16 +25,16 @@ gem build webscraping_ai.gemspec
 Then either install the gem locally:
 
 ```shell
-gem install ./webscraping_ai-3.1.3.gem
+gem install ./webscraping_ai-3.2.0.gem
 ```
 
-(for development, run `gem install --dev ./webscraping_ai-3.1.3.gem` to install the development dependencies)
+(for development, run `gem install --dev ./webscraping_ai-3.2.0.gem` to install the development dependencies)
 
 or publish the gem to a gem hosting service, e.g. [RubyGems](https://rubygems.org/).
 
 Finally add this to the Gemfile:
 
-    gem 'webscraping_ai', '~> 3.1.3'
+    gem 'webscraping_ai', '~> 3.2.0'
 
 ### Install from Git
 
@@ -67,17 +68,16 @@ end
 
 api_instance = WebScrapingAI::AIApi.new
 url = 'https://example.com' # String | URL of the target page.
+fields = { key: { key: 'inner_example'}} # Hash<String, String> | Object describing fields to extract from the page and their descriptions
 opts = {
-  question: 'What is the summary of this page content?', # String | Question or instructions to ask the LLM model about the target page.
-  context_limit: 4000, # Integer | Maximum number of tokens to use as context for the LLM model (4000 by default).
-  response_tokens: 100, # Integer | Maximum number of tokens to return in the LLM model response. The total context size (context_limit) includes the question, the target page content and the response, so this parameter reserves tokens for the response (see also on_context_limit).
-  on_context_limit: 'truncate', # String | What to do if the context_limit parameter is exceeded (truncate by default). The context is exceeded when the target page content is too long.
-  headers: { key: 3.56}, # Hash<String, String> | HTTP headers to pass to the target page. Can be specified either via a nested query parameter (...&headers[One]=value1&headers=[Another]=value2) or as a JSON encoded object (...&headers={\"One\": \"value1\", \"Another\": \"value2\"}).
+  headers: { key: { key: 'inner_example'}}, # Hash<String, String> | HTTP headers to pass to the target page. Can be specified either via a nested query parameter (...&headers[One]=value1&headers=[Another]=value2) or as a JSON encoded object (...&headers={\"One\": \"value1\", \"Another\": \"value2\"}).
   timeout: 10000, # Integer | Maximum web page retrieval time in ms. Increase it in case of timeout errors (10000 by default, maximum is 30000).
   js: true, # Boolean | Execute on-page JavaScript using a headless browser (true by default).
   js_timeout: 2000, # Integer | Maximum JavaScript rendering time in ms. Increase it in case if you see a loading indicator instead of data on the target page.
+  wait_for: 'wait_for_example', # String | CSS selector to wait for before returning the page content. Useful for pages with dynamic content loading. Overrides js_timeout.
   proxy: 'datacenter', # String | Type of proxy, use residential proxies if your site restricts traffic from datacenters (datacenter by default). Note that residential proxy requests are more expensive than datacenter, see the pricing page for details.
-  country: 'us', # String | Country of the proxy to use (US by default). Only available on Startup and Custom plans.
+  country: 'us', # String | Country of the proxy to use (US by default).
+  custom_proxy: 'custom_proxy_example', # String | Your own proxy URL to use instead of our built-in proxy pool in \"http://user:password@host:port\" format (<a target=\"_blank\" href=\"https://webscraping.ai/proxies/smartproxy\">Smartproxy</a> for example).
   device: 'desktop', # String | Type of device emulation.
   error_on_404: false, # Boolean | Return error on 404 HTTP status on the target page (false by default).
   error_on_redirect: false, # Boolean | Return error on redirect on the target page (false by default).
@@ -85,11 +85,11 @@ opts = {
 }
 
 begin
-  #Get an answer to a question about a given web page
-  result = api_instance.get_question(url, opts)
+  #Extract structured data fields from a web page
+  result = api_instance.get_fields(url, fields, opts)
   p result
 rescue WebScrapingAI::ApiError => e
-  puts "Exception when calling AIApi->get_question: #{e}"
+  puts "Exception when calling AIApi->get_fields: #{e}"
 end
 
 ```
@@ -100,6 +100,7 @@ All URIs are relative to *https://api.webscraping.ai*
 
 Class | Method | HTTP request | Description
 ------------ | ------------- | ------------- | -------------
+*WebScrapingAI::AIApi* | [**get_fields**](docs/AIApi.md#get_fields) | **GET** /ai/fields | Extract structured data fields from a web page
 *WebScrapingAI::AIApi* | [**get_question**](docs/AIApi.md#get_question) | **GET** /ai/question | Get an answer to a question about a given web page
 *WebScrapingAI::AccountApi* | [**account**](docs/AccountApi.md#account) | **GET** /account | Information about your account calls quota
 *WebScrapingAI::HTMLApi* | [**get_html**](docs/HTMLApi.md#get_html) | **GET** /html | Page HTML by URL

data/docs/AIApi.md

@@ -4,9 +4,109 @@ All URIs are relative to *https://api.webscraping.ai*
 
 | Method | HTTP request | Description |
 | ------ | ------------ | ----------- |
+| [**get_fields**](AIApi.md#get_fields) | **GET** /ai/fields | Extract structured data fields from a web page |
 | [**get_question**](AIApi.md#get_question) | **GET** /ai/question | Get an answer to a question about a given web page |
 
 
+## get_fields
+
+> Hash<String, String> get_fields(url, fields, opts)
+
+Extract structured data fields from a web page
+
+Returns structured data fields extracted from the webpage using an LLM model. Proxies and Chromium JavaScript rendering are used for page retrieval and processing.
+
+### Examples
+
+```ruby
+require 'time'
+require 'webscraping_ai'
+# setup authorization
+WebScrapingAI.configure do |config|
+  # Configure API key authorization: api_key
+  config.api_key['api_key'] = 'YOUR API KEY'
+  # Uncomment the following line to set a prefix for the API key, e.g. 'Bearer' (defaults to nil)
+  # config.api_key_prefix['api_key'] = 'Bearer'
+end
+
+api_instance = WebScrapingAI::AIApi.new
+url = 'https://example.com' # String | URL of the target page.
+fields = { key: { key: 'inner_example'}} # Hash<String, String> | Object describing fields to extract from the page and their descriptions
+opts = {
+  headers: { key: { key: 'inner_example'}}, # Hash<String, String> | HTTP headers to pass to the target page. Can be specified either via a nested query parameter (...&headers[One]=value1&headers=[Another]=value2) or as a JSON encoded object (...&headers={\"One\": \"value1\", \"Another\": \"value2\"}).
+  timeout: 10000, # Integer | Maximum web page retrieval time in ms. Increase it in case of timeout errors (10000 by default, maximum is 30000).
+  js: true, # Boolean | Execute on-page JavaScript using a headless browser (true by default).
+  js_timeout: 2000, # Integer | Maximum JavaScript rendering time in ms. Increase it in case if you see a loading indicator instead of data on the target page.
+  wait_for: 'wait_for_example', # String | CSS selector to wait for before returning the page content. Useful for pages with dynamic content loading. Overrides js_timeout.
+  proxy: 'datacenter', # String | Type of proxy, use residential proxies if your site restricts traffic from datacenters (datacenter by default). Note that residential proxy requests are more expensive than datacenter, see the pricing page for details.
+  country: 'us', # String | Country of the proxy to use (US by default).
+  custom_proxy: 'custom_proxy_example', # String | Your own proxy URL to use instead of our built-in proxy pool in \"http://user:password@host:port\" format (<a target=\"_blank\" href=\"https://webscraping.ai/proxies/smartproxy\">Smartproxy</a> for example).
+  device: 'desktop', # String | Type of device emulation.
+  error_on_404: false, # Boolean | Return error on 404 HTTP status on the target page (false by default).
+  error_on_redirect: false, # Boolean | Return error on redirect on the target page (false by default).
+  js_script: 'document.querySelector('button').click();' # String | Custom JavaScript code to execute on the target page.
+}
+
+begin
+  # Extract structured data fields from a web page
+  result = api_instance.get_fields(url, fields, opts)
+  p result
+rescue WebScrapingAI::ApiError => e
+  puts "Error when calling AIApi->get_fields: #{e}"
+end
+```
+
+#### Using the get_fields_with_http_info variant
+
+This returns an Array which contains the response data, status code and headers.
+
+> <Array(Hash&lt;String, String&gt;, Integer, Hash)> get_fields_with_http_info(url, fields, opts)
+
+```ruby
+begin
+  # Extract structured data fields from a web page
+  data, status_code, headers = api_instance.get_fields_with_http_info(url, fields, opts)
+  p status_code # => 2xx
+  p headers # => { ... }
+  p data # => Hash&lt;String, String&gt;
+rescue WebScrapingAI::ApiError => e
+  puts "Error when calling AIApi->get_fields_with_http_info: #{e}"
+end
+```
+
+### Parameters
+
+| Name | Type | Description | Notes |
+| ---- | ---- | ----------- | ----- |
+| **url** | **String** | URL of the target page. |  |
+| **fields** | [**Hash&lt;String, String&gt;**](String.md) | Object describing fields to extract from the page and their descriptions |  |
+| **headers** | [**Hash&lt;String, String&gt;**](String.md) | HTTP headers to pass to the target page. Can be specified either via a nested query parameter (...&amp;headers[One]&#x3D;value1&amp;headers&#x3D;[Another]&#x3D;value2) or as a JSON encoded object (...&amp;headers&#x3D;{\&quot;One\&quot;: \&quot;value1\&quot;, \&quot;Another\&quot;: \&quot;value2\&quot;}). | [optional] |
+| **timeout** | **Integer** | Maximum web page retrieval time in ms. Increase it in case of timeout errors (10000 by default, maximum is 30000). | [optional][default to 10000] |
+| **js** | **Boolean** | Execute on-page JavaScript using a headless browser (true by default). | [optional][default to true] |
+| **js_timeout** | **Integer** | Maximum JavaScript rendering time in ms. Increase it in case if you see a loading indicator instead of data on the target page. | [optional][default to 2000] |
+| **wait_for** | **String** | CSS selector to wait for before returning the page content. Useful for pages with dynamic content loading. Overrides js_timeout. | [optional] |
+| **proxy** | **String** | Type of proxy, use residential proxies if your site restricts traffic from datacenters (datacenter by default). Note that residential proxy requests are more expensive than datacenter, see the pricing page for details. | [optional][default to &#39;datacenter&#39;] |
+| **country** | **String** | Country of the proxy to use (US by default). | [optional][default to &#39;us&#39;] |
+| **custom_proxy** | **String** | Your own proxy URL to use instead of our built-in proxy pool in \&quot;http://user:password@host:port\&quot; format (&lt;a target&#x3D;\&quot;_blank\&quot; href&#x3D;\&quot;https://webscraping.ai/proxies/smartproxy\&quot;&gt;Smartproxy&lt;/a&gt; for example). | [optional] |
+| **device** | **String** | Type of device emulation. | [optional][default to &#39;desktop&#39;] |
+| **error_on_404** | **Boolean** | Return error on 404 HTTP status on the target page (false by default). | [optional][default to false] |
+| **error_on_redirect** | **Boolean** | Return error on redirect on the target page (false by default). | [optional][default to false] |
+| **js_script** | **String** | Custom JavaScript code to execute on the target page. | [optional] |
+
+### Return type
+
+**Hash&lt;String, String&gt;**
+
+### Authorization
+
+[api_key](../README.md#api_key)
+
+### HTTP request headers
+
+- **Content-Type**: Not defined
+- **Accept**: application/json
+
+
 ## get_question
 
 > String get_question(url, opts)
@@ -32,19 +132,19 @@ api_instance = WebScrapingAI::AIApi.new
 url = 'https://example.com' # String | URL of the target page.
 opts = {
   question: 'What is the summary of this page content?', # String | Question or instructions to ask the LLM model about the target page.
-  context_limit: 4000, # Integer | Maximum number of tokens to use as context for the LLM model (4000 by default).
-  response_tokens: 100, # Integer | Maximum number of tokens to return in the LLM model response. The total context size (context_limit) includes the question, the target page content and the response, so this parameter reserves tokens for the response (see also on_context_limit).
-  on_context_limit: 'truncate', # String | What to do if the context_limit parameter is exceeded (truncate by default). The context is exceeded when the target page content is too long.
-  headers: { key: 3.56}, # Hash<String, String> | HTTP headers to pass to the target page. Can be specified either via a nested query parameter (...&headers[One]=value1&headers=[Another]=value2) or as a JSON encoded object (...&headers={\"One\": \"value1\", \"Another\": \"value2\"}).
+  headers: { key: { key: 'inner_example'}}, # Hash<String, String> | HTTP headers to pass to the target page. Can be specified either via a nested query parameter (...&headers[One]=value1&headers=[Another]=value2) or as a JSON encoded object (...&headers={\"One\": \"value1\", \"Another\": \"value2\"}).
   timeout: 10000, # Integer | Maximum web page retrieval time in ms. Increase it in case of timeout errors (10000 by default, maximum is 30000).
   js: true, # Boolean | Execute on-page JavaScript using a headless browser (true by default).
   js_timeout: 2000, # Integer | Maximum JavaScript rendering time in ms. Increase it in case if you see a loading indicator instead of data on the target page.
+  wait_for: 'wait_for_example', # String | CSS selector to wait for before returning the page content. Useful for pages with dynamic content loading. Overrides js_timeout.
   proxy: 'datacenter', # String | Type of proxy, use residential proxies if your site restricts traffic from datacenters (datacenter by default). Note that residential proxy requests are more expensive than datacenter, see the pricing page for details.
-  country: 'us', # String | Country of the proxy to use (US by default). Only available on Startup and Custom plans.
+  country: 'us', # String | Country of the proxy to use (US by default).
+  custom_proxy: 'custom_proxy_example', # String | Your own proxy URL to use instead of our built-in proxy pool in \"http://user:password@host:port\" format (<a target=\"_blank\" href=\"https://webscraping.ai/proxies/smartproxy\">Smartproxy</a> for example).
   device: 'desktop', # String | Type of device emulation.
   error_on_404: false, # Boolean | Return error on 404 HTTP status on the target page (false by default).
   error_on_redirect: false, # Boolean | Return error on redirect on the target page (false by default).
-  js_script: 'document.querySelector('button').click();' # String | Custom JavaScript code to execute on the target page.
+  js_script: 'document.querySelector('button').click();', # String | Custom JavaScript code to execute on the target page.
+  format: 'json' # String | Format of the response (text by default). \"json\" will return a JSON object with the response, \"text\" will return a plain text/HTML response.
 }
 
 begin
@@ -80,19 +180,19 @@ end
 | ---- | ---- | ----------- | ----- |
 | **url** | **String** | URL of the target page. |  |
 | **question** | **String** | Question or instructions to ask the LLM model about the target page. | [optional] |
-| **context_limit** | **Integer** | Maximum number of tokens to use as context for the LLM model (4000 by default). | [optional][default to 4000] |
-| **response_tokens** | **Integer** | Maximum number of tokens to return in the LLM model response. The total context size (context_limit) includes the question, the target page content and the response, so this parameter reserves tokens for the response (see also on_context_limit). | [optional][default to 100] |
-| **on_context_limit** | **String** | What to do if the context_limit parameter is exceeded (truncate by default). The context is exceeded when the target page content is too long. | [optional][default to &#39;error&#39;] |
 | **headers** | [**Hash&lt;String, String&gt;**](String.md) | HTTP headers to pass to the target page. Can be specified either via a nested query parameter (...&amp;headers[One]&#x3D;value1&amp;headers&#x3D;[Another]&#x3D;value2) or as a JSON encoded object (...&amp;headers&#x3D;{\&quot;One\&quot;: \&quot;value1\&quot;, \&quot;Another\&quot;: \&quot;value2\&quot;}). | [optional] |
 | **timeout** | **Integer** | Maximum web page retrieval time in ms. Increase it in case of timeout errors (10000 by default, maximum is 30000). | [optional][default to 10000] |
 | **js** | **Boolean** | Execute on-page JavaScript using a headless browser (true by default). | [optional][default to true] |
 | **js_timeout** | **Integer** | Maximum JavaScript rendering time in ms. Increase it in case if you see a loading indicator instead of data on the target page. | [optional][default to 2000] |
+| **wait_for** | **String** | CSS selector to wait for before returning the page content. Useful for pages with dynamic content loading. Overrides js_timeout. | [optional] |
 | **proxy** | **String** | Type of proxy, use residential proxies if your site restricts traffic from datacenters (datacenter by default). Note that residential proxy requests are more expensive than datacenter, see the pricing page for details. | [optional][default to &#39;datacenter&#39;] |
-| **country** | **String** | Country of the proxy to use (US by default). Only available on Startup and Custom plans. | [optional][default to &#39;us&#39;] |
+| **country** | **String** | Country of the proxy to use (US by default). | [optional][default to &#39;us&#39;] |
+| **custom_proxy** | **String** | Your own proxy URL to use instead of our built-in proxy pool in \&quot;http://user:password@host:port\&quot; format (&lt;a target&#x3D;\&quot;_blank\&quot; href&#x3D;\&quot;https://webscraping.ai/proxies/smartproxy\&quot;&gt;Smartproxy&lt;/a&gt; for example). | [optional] |
 | **device** | **String** | Type of device emulation. | [optional][default to &#39;desktop&#39;] |
 | **error_on_404** | **Boolean** | Return error on 404 HTTP status on the target page (false by default). | [optional][default to false] |
 | **error_on_redirect** | **Boolean** | Return error on redirect on the target page (false by default). | [optional][default to false] |
 | **js_script** | **String** | Custom JavaScript code to execute on the target page. | [optional] |
+| **format** | **String** | Format of the response (text by default). \&quot;json\&quot; will return a JSON object with the response, \&quot;text\&quot; will return a plain text/HTML response. | [optional][default to &#39;json&#39;] |
 
 ### Return type
 

data/docs/Account.md

@@ -4,6 +4,7 @@
 
 | Name | Type | Description | Notes |
 | ---- | ---- | ----------- | ----- |
+| **email** | **String** | Your account email | [optional] |
 | **remaining_api_calls** | **Integer** | Remaining API credits quota | [optional] |
 | **resets_at** | **Integer** | Next billing cycle start time (UNIX timestamp) | [optional] |
 | **remaining_concurrency** | **Integer** | Remaining concurrent requests | [optional] |
@@ -14,6 +15,7 @@
 require 'webscraping_ai'
 
 instance = WebScrapingAI::Account.new(
+  email: null,
   remaining_api_calls: null,
   resets_at: null,
   remaining_concurrency: null

data/docs/HTMLApi.md

@@ -31,17 +31,20 @@ end
 api_instance = WebScrapingAI::HTMLApi.new
 url = 'https://example.com' # String | URL of the target page.
 opts = {
-  headers: { key: 3.56}, # Hash<String, String> | HTTP headers to pass to the target page. Can be specified either via a nested query parameter (...&headers[One]=value1&headers=[Another]=value2) or as a JSON encoded object (...&headers={\"One\": \"value1\", \"Another\": \"value2\"}).
+  headers: { key: { key: 'inner_example'}}, # Hash<String, String> | HTTP headers to pass to the target page. Can be specified either via a nested query parameter (...&headers[One]=value1&headers=[Another]=value2) or as a JSON encoded object (...&headers={\"One\": \"value1\", \"Another\": \"value2\"}).
   timeout: 10000, # Integer | Maximum web page retrieval time in ms. Increase it in case of timeout errors (10000 by default, maximum is 30000).
   js: true, # Boolean | Execute on-page JavaScript using a headless browser (true by default).
   js_timeout: 2000, # Integer | Maximum JavaScript rendering time in ms. Increase it in case if you see a loading indicator instead of data on the target page.
+  wait_for: 'wait_for_example', # String | CSS selector to wait for before returning the page content. Useful for pages with dynamic content loading. Overrides js_timeout.
   proxy: 'datacenter', # String | Type of proxy, use residential proxies if your site restricts traffic from datacenters (datacenter by default). Note that residential proxy requests are more expensive than datacenter, see the pricing page for details.
-  country: 'us', # String | Country of the proxy to use (US by default). Only available on Startup and Custom plans.
+  country: 'us', # String | Country of the proxy to use (US by default).
+  custom_proxy: 'custom_proxy_example', # String | Your own proxy URL to use instead of our built-in proxy pool in \"http://user:password@host:port\" format (<a target=\"_blank\" href=\"https://webscraping.ai/proxies/smartproxy\">Smartproxy</a> for example).
   device: 'desktop', # String | Type of device emulation.
   error_on_404: false, # Boolean | Return error on 404 HTTP status on the target page (false by default).
   error_on_redirect: false, # Boolean | Return error on redirect on the target page (false by default).
   js_script: 'document.querySelector('button').click();', # String | Custom JavaScript code to execute on the target page.
-  return_script_result: false # Boolean | Return result of the custom JavaScript code (js_script parameter) execution on the target page (false by default, page HTML will be returned).
+  return_script_result: false, # Boolean | Return result of the custom JavaScript code (js_script parameter) execution on the target page (false by default, page HTML will be returned).
+  format: 'json' # String | Format of the response (text by default). \"json\" will return a JSON object with the response, \"text\" will return a plain text/HTML response.
 }
 
 begin
@@ -80,13 +83,16 @@ end
 | **timeout** | **Integer** | Maximum web page retrieval time in ms. Increase it in case of timeout errors (10000 by default, maximum is 30000). | [optional][default to 10000] |
 | **js** | **Boolean** | Execute on-page JavaScript using a headless browser (true by default). | [optional][default to true] |
 | **js_timeout** | **Integer** | Maximum JavaScript rendering time in ms. Increase it in case if you see a loading indicator instead of data on the target page. | [optional][default to 2000] |
+| **wait_for** | **String** | CSS selector to wait for before returning the page content. Useful for pages with dynamic content loading. Overrides js_timeout. | [optional] |
 | **proxy** | **String** | Type of proxy, use residential proxies if your site restricts traffic from datacenters (datacenter by default). Note that residential proxy requests are more expensive than datacenter, see the pricing page for details. | [optional][default to &#39;datacenter&#39;] |
-| **country** | **String** | Country of the proxy to use (US by default). Only available on Startup and Custom plans. | [optional][default to &#39;us&#39;] |
+| **country** | **String** | Country of the proxy to use (US by default). | [optional][default to &#39;us&#39;] |
+| **custom_proxy** | **String** | Your own proxy URL to use instead of our built-in proxy pool in \&quot;http://user:password@host:port\&quot; format (&lt;a target&#x3D;\&quot;_blank\&quot; href&#x3D;\&quot;https://webscraping.ai/proxies/smartproxy\&quot;&gt;Smartproxy&lt;/a&gt; for example). | [optional] |
 | **device** | **String** | Type of device emulation. | [optional][default to &#39;desktop&#39;] |
 | **error_on_404** | **Boolean** | Return error on 404 HTTP status on the target page (false by default). | [optional][default to false] |
 | **error_on_redirect** | **Boolean** | Return error on redirect on the target page (false by default). | [optional][default to false] |
 | **js_script** | **String** | Custom JavaScript code to execute on the target page. | [optional] |
 | **return_script_result** | **Boolean** | Return result of the custom JavaScript code (js_script parameter) execution on the target page (false by default, page HTML will be returned). | [optional][default to false] |
+| **format** | **String** | Format of the response (text by default). \&quot;json\&quot; will return a JSON object with the response, \&quot;text\&quot; will return a plain text/HTML response. | [optional][default to &#39;json&#39;] |
 
 ### Return type
 

data/docs/SelectedHTMLApi.md

@@ -33,16 +33,19 @@ api_instance = WebScrapingAI::SelectedHTMLApi.new
 url = 'https://example.com' # String | URL of the target page.
 opts = {
   selector: 'h1', # String | CSS selector (null by default, returns whole page HTML)
-  headers: { key: 3.56}, # Hash<String, String> | HTTP headers to pass to the target page. Can be specified either via a nested query parameter (...&headers[One]=value1&headers=[Another]=value2) or as a JSON encoded object (...&headers={\"One\": \"value1\", \"Another\": \"value2\"}).
+  headers: { key: { key: 'inner_example'}}, # Hash<String, String> | HTTP headers to pass to the target page. Can be specified either via a nested query parameter (...&headers[One]=value1&headers=[Another]=value2) or as a JSON encoded object (...&headers={\"One\": \"value1\", \"Another\": \"value2\"}).
   timeout: 10000, # Integer | Maximum web page retrieval time in ms. Increase it in case of timeout errors (10000 by default, maximum is 30000).
   js: true, # Boolean | Execute on-page JavaScript using a headless browser (true by default).
   js_timeout: 2000, # Integer | Maximum JavaScript rendering time in ms. Increase it in case if you see a loading indicator instead of data on the target page.
+  wait_for: 'wait_for_example', # String | CSS selector to wait for before returning the page content. Useful for pages with dynamic content loading. Overrides js_timeout.
   proxy: 'datacenter', # String | Type of proxy, use residential proxies if your site restricts traffic from datacenters (datacenter by default). Note that residential proxy requests are more expensive than datacenter, see the pricing page for details.
-  country: 'us', # String | Country of the proxy to use (US by default). Only available on Startup and Custom plans.
+  country: 'us', # String | Country of the proxy to use (US by default).
+  custom_proxy: 'custom_proxy_example', # String | Your own proxy URL to use instead of our built-in proxy pool in \"http://user:password@host:port\" format (<a target=\"_blank\" href=\"https://webscraping.ai/proxies/smartproxy\">Smartproxy</a> for example).
   device: 'desktop', # String | Type of device emulation.
   error_on_404: false, # Boolean | Return error on 404 HTTP status on the target page (false by default).
   error_on_redirect: false, # Boolean | Return error on redirect on the target page (false by default).
-  js_script: 'document.querySelector('button').click();' # String | Custom JavaScript code to execute on the target page.
+  js_script: 'document.querySelector('button').click();', # String | Custom JavaScript code to execute on the target page.
+  format: 'json' # String | Format of the response (text by default). \"json\" will return a JSON object with the response, \"text\" will return a plain text/HTML response.
 }
 
 begin
@@ -82,12 +85,15 @@ end
 | **timeout** | **Integer** | Maximum web page retrieval time in ms. Increase it in case of timeout errors (10000 by default, maximum is 30000). | [optional][default to 10000] |
 | **js** | **Boolean** | Execute on-page JavaScript using a headless browser (true by default). | [optional][default to true] |
 | **js_timeout** | **Integer** | Maximum JavaScript rendering time in ms. Increase it in case if you see a loading indicator instead of data on the target page. | [optional][default to 2000] |
+| **wait_for** | **String** | CSS selector to wait for before returning the page content. Useful for pages with dynamic content loading. Overrides js_timeout. | [optional] |
 | **proxy** | **String** | Type of proxy, use residential proxies if your site restricts traffic from datacenters (datacenter by default). Note that residential proxy requests are more expensive than datacenter, see the pricing page for details. | [optional][default to &#39;datacenter&#39;] |
-| **country** | **String** | Country of the proxy to use (US by default). Only available on Startup and Custom plans. | [optional][default to &#39;us&#39;] |
+| **country** | **String** | Country of the proxy to use (US by default). | [optional][default to &#39;us&#39;] |
+| **custom_proxy** | **String** | Your own proxy URL to use instead of our built-in proxy pool in \&quot;http://user:password@host:port\&quot; format (&lt;a target&#x3D;\&quot;_blank\&quot; href&#x3D;\&quot;https://webscraping.ai/proxies/smartproxy\&quot;&gt;Smartproxy&lt;/a&gt; for example). | [optional] |
 | **device** | **String** | Type of device emulation. | [optional][default to &#39;desktop&#39;] |
 | **error_on_404** | **Boolean** | Return error on 404 HTTP status on the target page (false by default). | [optional][default to false] |
 | **error_on_redirect** | **Boolean** | Return error on redirect on the target page (false by default). | [optional][default to false] |
 | **js_script** | **String** | Custom JavaScript code to execute on the target page. | [optional] |
+| **format** | **String** | Format of the response (text by default). \&quot;json\&quot; will return a JSON object with the response, \&quot;text\&quot; will return a plain text/HTML response. | [optional][default to &#39;json&#39;] |
 
 ### Return type
 
@@ -128,12 +134,14 @@ api_instance = WebScrapingAI::SelectedHTMLApi.new
 url = 'https://example.com' # String | URL of the target page.
 opts = {
   selectors: ['inner_example'], # Array<String> | Multiple CSS selectors (null by default, returns whole page HTML)
-  headers: { key: 3.56}, # Hash<String, String> | HTTP headers to pass to the target page. Can be specified either via a nested query parameter (...&headers[One]=value1&headers=[Another]=value2) or as a JSON encoded object (...&headers={\"One\": \"value1\", \"Another\": \"value2\"}).
+  headers: { key: { key: 'inner_example'}}, # Hash<String, String> | HTTP headers to pass to the target page. Can be specified either via a nested query parameter (...&headers[One]=value1&headers=[Another]=value2) or as a JSON encoded object (...&headers={\"One\": \"value1\", \"Another\": \"value2\"}).
   timeout: 10000, # Integer | Maximum web page retrieval time in ms. Increase it in case of timeout errors (10000 by default, maximum is 30000).
   js: true, # Boolean | Execute on-page JavaScript using a headless browser (true by default).
   js_timeout: 2000, # Integer | Maximum JavaScript rendering time in ms. Increase it in case if you see a loading indicator instead of data on the target page.
+  wait_for: 'wait_for_example', # String | CSS selector to wait for before returning the page content. Useful for pages with dynamic content loading. Overrides js_timeout.
   proxy: 'datacenter', # String | Type of proxy, use residential proxies if your site restricts traffic from datacenters (datacenter by default). Note that residential proxy requests are more expensive than datacenter, see the pricing page for details.
-  country: 'us', # String | Country of the proxy to use (US by default). Only available on Startup and Custom plans.
+  country: 'us', # String | Country of the proxy to use (US by default).
+  custom_proxy: 'custom_proxy_example', # String | Your own proxy URL to use instead of our built-in proxy pool in \"http://user:password@host:port\" format (<a target=\"_blank\" href=\"https://webscraping.ai/proxies/smartproxy\">Smartproxy</a> for example).
   device: 'desktop', # String | Type of device emulation.
   error_on_404: false, # Boolean | Return error on 404 HTTP status on the target page (false by default).
   error_on_redirect: false, # Boolean | Return error on redirect on the target page (false by default).
@@ -177,8 +185,10 @@ end
 | **timeout** | **Integer** | Maximum web page retrieval time in ms. Increase it in case of timeout errors (10000 by default, maximum is 30000). | [optional][default to 10000] |
 | **js** | **Boolean** | Execute on-page JavaScript using a headless browser (true by default). | [optional][default to true] |
 | **js_timeout** | **Integer** | Maximum JavaScript rendering time in ms. Increase it in case if you see a loading indicator instead of data on the target page. | [optional][default to 2000] |
+| **wait_for** | **String** | CSS selector to wait for before returning the page content. Useful for pages with dynamic content loading. Overrides js_timeout. | [optional] |
 | **proxy** | **String** | Type of proxy, use residential proxies if your site restricts traffic from datacenters (datacenter by default). Note that residential proxy requests are more expensive than datacenter, see the pricing page for details. | [optional][default to &#39;datacenter&#39;] |
-| **country** | **String** | Country of the proxy to use (US by default). Only available on Startup and Custom plans. | [optional][default to &#39;us&#39;] |
+| **country** | **String** | Country of the proxy to use (US by default). | [optional][default to &#39;us&#39;] |
+| **custom_proxy** | **String** | Your own proxy URL to use instead of our built-in proxy pool in \&quot;http://user:password@host:port\&quot; format (&lt;a target&#x3D;\&quot;_blank\&quot; href&#x3D;\&quot;https://webscraping.ai/proxies/smartproxy\&quot;&gt;Smartproxy&lt;/a&gt; for example). | [optional] |
 | **device** | **String** | Type of device emulation. | [optional][default to &#39;desktop&#39;] |
 | **error_on_404** | **Boolean** | Return error on 404 HTTP status on the target page (false by default). | [optional][default to false] |
 | **error_on_redirect** | **Boolean** | Return error on redirect on the target page (false by default). | [optional][default to false] |

data/docs/TextApi.md

@@ -13,7 +13,7 @@ All URIs are relative to *https://api.webscraping.ai*
 
 Page text by URL
 
-Returns the visible text content of a webpage specified by the URL. Can be used to feed data to GPT or other LLM models. The response can be in plain text, JSON, or XML format based on the text_format parameter. Proxies and Chromium JavaScript rendering are used for page retrieval and processing. Returns JSON on error.
+Returns the visible text content of a webpage specified by the URL. Can be used to feed data to LLM models. The response can be in plain text, JSON, or XML format based on the text_format parameter. Proxies and Chromium JavaScript rendering are used for page retrieval and processing. Returns JSON on error.
 
 ### Examples
 
@@ -33,12 +33,14 @@ url = 'https://example.com' # String | URL of the target page.
 opts = {
   text_format: 'plain', # String | Format of the text response (plain by default). \"plain\" will return only the page body text. \"json\" and \"xml\" will return a json/xml with \"title\", \"description\" and \"content\" keys.
   return_links: false, # Boolean | [Works only with text_format=json] Return links from the page body text (false by default). Useful for building web crawlers.
-  headers: { key: 3.56}, # Hash<String, String> | HTTP headers to pass to the target page. Can be specified either via a nested query parameter (...&headers[One]=value1&headers=[Another]=value2) or as a JSON encoded object (...&headers={\"One\": \"value1\", \"Another\": \"value2\"}).
+  headers: { key: { key: 'inner_example'}}, # Hash<String, String> | HTTP headers to pass to the target page. Can be specified either via a nested query parameter (...&headers[One]=value1&headers=[Another]=value2) or as a JSON encoded object (...&headers={\"One\": \"value1\", \"Another\": \"value2\"}).
   timeout: 10000, # Integer | Maximum web page retrieval time in ms. Increase it in case of timeout errors (10000 by default, maximum is 30000).
   js: true, # Boolean | Execute on-page JavaScript using a headless browser (true by default).
   js_timeout: 2000, # Integer | Maximum JavaScript rendering time in ms. Increase it in case if you see a loading indicator instead of data on the target page.
+  wait_for: 'wait_for_example', # String | CSS selector to wait for before returning the page content. Useful for pages with dynamic content loading. Overrides js_timeout.
   proxy: 'datacenter', # String | Type of proxy, use residential proxies if your site restricts traffic from datacenters (datacenter by default). Note that residential proxy requests are more expensive than datacenter, see the pricing page for details.
-  country: 'us', # String | Country of the proxy to use (US by default). Only available on Startup and Custom plans.
+  country: 'us', # String | Country of the proxy to use (US by default).
+  custom_proxy: 'custom_proxy_example', # String | Your own proxy URL to use instead of our built-in proxy pool in \"http://user:password@host:port\" format (<a target=\"_blank\" href=\"https://webscraping.ai/proxies/smartproxy\">Smartproxy</a> for example).
   device: 'desktop', # String | Type of device emulation.
   error_on_404: false, # Boolean | Return error on 404 HTTP status on the target page (false by default).
   error_on_redirect: false, # Boolean | Return error on redirect on the target page (false by default).
@@ -83,8 +85,10 @@ end
 | **timeout** | **Integer** | Maximum web page retrieval time in ms. Increase it in case of timeout errors (10000 by default, maximum is 30000). | [optional][default to 10000] |
 | **js** | **Boolean** | Execute on-page JavaScript using a headless browser (true by default). | [optional][default to true] |
 | **js_timeout** | **Integer** | Maximum JavaScript rendering time in ms. Increase it in case if you see a loading indicator instead of data on the target page. | [optional][default to 2000] |
+| **wait_for** | **String** | CSS selector to wait for before returning the page content. Useful for pages with dynamic content loading. Overrides js_timeout. | [optional] |
 | **proxy** | **String** | Type of proxy, use residential proxies if your site restricts traffic from datacenters (datacenter by default). Note that residential proxy requests are more expensive than datacenter, see the pricing page for details. | [optional][default to &#39;datacenter&#39;] |
-| **country** | **String** | Country of the proxy to use (US by default). Only available on Startup and Custom plans. | [optional][default to &#39;us&#39;] |
+| **country** | **String** | Country of the proxy to use (US by default). | [optional][default to &#39;us&#39;] |
+| **custom_proxy** | **String** | Your own proxy URL to use instead of our built-in proxy pool in \&quot;http://user:password@host:port\&quot; format (&lt;a target&#x3D;\&quot;_blank\&quot; href&#x3D;\&quot;https://webscraping.ai/proxies/smartproxy\&quot;&gt;Smartproxy&lt;/a&gt; for example). | [optional] |
 | **device** | **String** | Type of device emulation. | [optional][default to &#39;desktop&#39;] |
 | **error_on_404** | **Boolean** | Return error on 404 HTTP status on the target page (false by default). | [optional][default to false] |
 | **error_on_redirect** | **Boolean** | Return error on redirect on the target page (false by default). | [optional][default to false] |

data/lib/webscraping_ai/api/account_api.rb

@@ -1,12 +1,12 @@
 =begin
 #WebScraping.AI
 
-#WebScraping.AI scraping API provides GPT-powered tools with Chromium JavaScript rendering, rotating proxies, and built-in HTML parsing.
+#WebScraping.AI scraping API provides LLM-powered tools with Chromium JavaScript rendering, rotating proxies, and built-in HTML parsing.
 
-The version of the OpenAPI document: 3.1.3
+The version of the OpenAPI document: 3.2.0
 Contact: support@webscraping.ai
 Generated by: https://openapi-generator.tech
-OpenAPI Generator version: 7.2.0
+Generator version: 7.11.0
 
 =end
 
@@ -45,7 +45,7 @@ module WebScrapingAI
       # header parameters
       header_params = opts[:header_params] || {}
       # HTTP header 'Accept' (if needed)
-      header_params['Accept'] = @api_client.select_header_accept(['application/json'])
+      header_params['Accept'] = @api_client.select_header_accept(['application/json']) unless header_params['Accept']
 
       # form parameters
       form_params = opts[:form_params] || {}