webscraping_ai 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 87c7c3987da110e442c1a4af9c64faf489e1357e19e961d9e8ccf064986b2cf0
+  data.tar.gz: 7705c46a0f70dccbbc8c3e51467e37c84faab243330bbeb6069733fa732a3afa
+SHA512:
+  metadata.gz: 33d32382161eb7cdbacfb27ffb75c0d758b507e7945adc8c9d418fee2ff6bda748d1b5671ed7c2f430af3fc522537856a7a2ef85fd844e92272203d3d8f06341
+  data.tar.gz: 3482be7567deceb06613d2cf82a9d56d1cc7c9ef51b8a27fde3d2df283c91f3d695fceb9dca8adc96a8f48f81c1ad624c9419f1eb700f533e932285e9e512da8

data/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+
+gemspec
+
+group :development, :test do
+  gem 'rake', '~> 12.0.0'
+  gem 'pry-byebug'
+  gem 'rubocop', '~> 0.66.0'
+end

data/Gemfile.lock

@@ -0,0 +1,70 @@
+PATH
+  remote: .
+  specs:
+    webscraping_ai (1.0.0)
+      json (~> 2.1, >= 2.1.0)
+      typhoeus (~> 1.0, >= 1.0.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.0)
+    byebug (11.1.1)
+    coderay (1.1.2)
+    diff-lcs (1.3)
+    ethon (0.12.0)
+      ffi (>= 1.3.0)
+    ffi (1.12.2)
+    jaro_winkler (1.5.4)
+    json (2.3.0)
+    method_source (1.0.0)
+    parallel (1.19.1)
+    parser (2.7.0.5)
+      ast (~> 2.4.0)
+    pry (0.13.0)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    pry-byebug (3.9.0)
+      byebug (~> 11.0)
+      pry (~> 0.13.0)
+    psych (3.1.0)
+    rainbow (3.0.0)
+    rake (12.0.0)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.1)
+      rspec-support (~> 3.9.1)
+    rspec-expectations (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.2)
+    rubocop (0.66.0)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.5, != 2.5.1.1)
+      psych (>= 3.1.0)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 1.6)
+    ruby-progressbar (1.10.1)
+    typhoeus (1.3.1)
+      ethon (>= 0.9.0)
+    unicode-display_width (1.5.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  pry-byebug
+  rake (~> 12.0.0)
+  rspec (~> 3.6, >= 3.6.0)
+  rubocop (~> 0.66.0)
+  webscraping_ai!
+
+BUNDLED WITH
+   2.1.2

data/README.md

@@ -0,0 +1,78 @@
+# webscraping_ai
+
+WebScrapingAI - the Ruby gem for the WebScraping.AI
+
+A client for https://webscraping.ai API. It provides Chrome JS rendering, rotating proxies and HTML parsing for web scraping.
+
+This SDK is automatically generated by the [OpenAPI Generator](https://openapi-generator.tech) project:
+
+- API version: 1.0.0
+- Package version: 1.0.0
+- Build package: org.openapitools.codegen.languages.RubyClientCodegen
+
+## Installation
+
+### Install from RubyGems
+
+Add the following in the Gemfile:
+
+    gem 'webscraping_ai'
+
+## Getting Started
+
+Please follow the [installation](#installation) procedure and then run the following code:
+
+```ruby
+# Load the gem
+require 'webscraping_ai'
+
+# Setup authorization
+WebScrapingAI.configure do |config|
+  # Configure API key authorization: api_key
+  config.api_key['api_key'] = 'test-api-key'
+end
+
+api_instance = WebScrapingAI::HtmlApi.new
+url = 'https://example.com' # String | URL of the page to get
+opts = {
+  selector: 'html', # String | CSS selector to get a part of the page (null by default, returns whole page HTML)
+  outer_html: false, # Boolean | Return outer HTML of the selected element (false by default, returns inner HTML)
+  proxy: 'US', # String | Proxy country code, for geotargeting (US by default)
+  disable_js: false, # Boolean | Disable JS execution (false by default)
+  inline_css: false # Boolean | Inline included CSS files to make page viewable on other domains (false by default)
+}
+
+begin
+  #Get page HTML by URL (renders JS in Chrome and uses rotating proxies)
+  result = api_instance.get_page(url, opts)
+  p result
+rescue WebScrapingAI::ApiError => e
+  puts "Exception when calling HtmlApi->get_page: #{e}"
+end
+
+```
+
+## Documentation for API Endpoints
+
+All URIs are relative to *https://webscraping.ai/api*
+
+Class | Method | HTTP request | Description
+------------ | ------------- | ------------- | -------------
+*WebScrapingAI::HtmlApi* | [**get_page**](docs/HtmlApi.md#get_page) | **GET** / | Get page HTML by URL (renders JS in Chrome and uses rotating proxies)
+
+
+## Documentation for Models
+
+ - [WebScrapingAI::ScrappedPage](docs/ScrappedPage.md)
+
+
+## Documentation for Authorization
+
+
+### api_key
+
+
+- **Type**: API key
+- **API key parameter name**: api_key
+- **Location**: URL query string
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+
+begin
+  require 'rspec/core/rake_task'
+
+  RSpec::Core::RakeTask.new(:spec)
+  task default: :spec
+rescue LoadError
+  # no rspec available
+end

data/docs/HtmlApi.md

@@ -0,0 +1,73 @@
+# WebScrapingAI::HtmlApi
+
+All URIs are relative to *https://webscraping.ai/api*
+
+Method | HTTP request | Description
+------------- | ------------- | -------------
+[**get_page**](HtmlApi.md#get_page) | **GET** / | Get page HTML by URL (renders JS in Chrome and uses rotating proxies)
+
+
+
+## get_page
+
+> ScrappedPage get_page(url, opts)
+
+Get page HTML by URL (renders JS in Chrome and uses rotating proxies)
+
+### Example
+
+```ruby
+# load the gem
+require 'webscraping_ai'
+# setup authorization
+WebScrapingAI.configure do |config|
+  # Configure API key authorization: api_key
+  config.api_key['api_key'] = 'YOUR API KEY'
+  # Uncomment the following line to set a prefix for the API key, e.g. 'Bearer' (defaults to nil)
+  #config.api_key_prefix['api_key'] = 'Bearer'
+end
+
+api_instance = WebScrapingAI::HtmlApi.new
+url = 'https://example.com' # String | URL of the page to get
+opts = {
+  selector: 'html', # String | CSS selector to get a part of the page (null by default, returns whole page HTML)
+  outer_html: false, # Boolean | Return outer HTML of the selected element (false by default, returns inner HTML)
+  proxy: 'US', # String | Proxy country code, for geotargeting (US by default)
+  disable_js: false, # Boolean | Disable JS execution (false by default)
+  inline_css: false # Boolean | Inline included CSS files to make page viewable on other domains (false by default)
+}
+
+begin
+  #Get page HTML by URL (renders JS in Chrome and uses rotating proxies)
+  result = api_instance.get_page(url, opts)
+  p result
+rescue WebScrapingAI::ApiError => e
+  puts "Exception when calling HtmlApi->get_page: #{e}"
+end
+```
+
+### Parameters
+
+
+Name | Type | Description  | Notes
+------------- | ------------- | ------------- | -------------
+ **url** | **String**| URL of the page to get | 
+ **selector** | **String**| CSS selector to get a part of the page (null by default, returns whole page HTML) | [optional] 
+ **outer_html** | **Boolean**| Return outer HTML of the selected element (false by default, returns inner HTML) | [optional] 
+ **proxy** | **String**| Proxy country code, for geotargeting (US by default) | [optional] 
+ **disable_js** | **Boolean**| Disable JS execution (false by default) | [optional] 
+ **inline_css** | **Boolean**| Inline included CSS files to make page viewable on other domains (false by default) | [optional] 
+
+### Return type
+
+[**ScrappedPage**](ScrappedPage.md)
+
+### Authorization
+
+[api_key](../README.md#api_key)
+
+### HTTP request headers
+
+- **Content-Type**: Not defined
+- **Accept**: application/json
+

data/docs/ScrappedPage.md

@@ -0,0 +1,23 @@
+# WebScrapingAI::ScrappedPage
+
+## Properties
+
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+**size_bytes** | **Integer** | Page HTML content size in bytes | [optional] 
+**html** | **String** | HTML of the full page or a selected area | [optional] 
+**status** | **Integer** | Response HTTP status code (200, 404, 302, etc) | [optional] 
+**status_message** | **String** | Response HTTP status message | [optional] 
+
+## Code Sample
+
+```ruby
+require 'WebScrapingAI'
+
+instance = WebScrapingAI::ScrappedPage.new(size_bytes: null,
+                                 html: null,
+                                 status: null,
+                                 status_message: null)
+```
+
+

data/git_push.sh

@@ -0,0 +1,58 @@
+#!/bin/sh
+# ref: https://help.github.com/articles/adding-an-existing-project-to-github-using-the-command-line/
+#
+# Usage example: /bin/sh ./git_push.sh wing328 openapi-pestore-perl "minor update" "gitlab.com"
+
+git_user_id=$1
+git_repo_id=$2
+release_note=$3
+git_host=$4
+
+if [ "$git_host" = "" ]; then
+    git_host="github.com"
+    echo "[INFO] No command line input provided. Set \$git_host to $git_host"
+fi
+
+if [ "$git_user_id" = "" ]; then
+    git_user_id="webscraping-ai"
+    echo "[INFO] No command line input provided. Set \$git_user_id to $git_user_id"
+fi
+
+if [ "$git_repo_id" = "" ]; then
+    git_repo_id="webscraping-ai-ruby"
+    echo "[INFO] No command line input provided. Set \$git_repo_id to $git_repo_id"
+fi
+
+if [ "$release_note" = "" ]; then
+    release_note="Minor update"
+    echo "[INFO] No command line input provided. Set \$release_note to $release_note"
+fi
+
+# Initialize the local directory as a Git repository
+git init
+
+# Adds the files in the local repository and stages them for commit.
+git add .
+
+# Commits the tracked changes and prepares them to be pushed to a remote repository.
+git commit -m "$release_note"
+
+# Sets the new remote
+git_remote=`git remote`
+if [ "$git_remote" = "" ]; then # git remote not defined
+
+    if [ "$GIT_TOKEN" = "" ]; then
+        echo "[INFO] \$GIT_TOKEN (environment variable) is not set. Using the git credential in your environment."
+        git remote add origin https://${git_host}/${git_user_id}/${git_repo_id}.git
+    else
+        git remote add origin https://${git_user_id}:${GIT_TOKEN}@${git_host}/${git_user_id}/${git_repo_id}.git
+    fi
+
+fi
+
+git pull origin master
+
+# Pushes (Forces) the changes in the local repository up to the remote repository
+echo "Git pushing to https://${git_host}/${git_user_id}/${git_repo_id}.git"
+git push origin master 2>&1 | grep -v 'To https'
+

data/lib/webscraping_ai/api/html_api.rb

@@ -0,0 +1,98 @@
+=begin
+#WebScraping.AI
+
+#A client for https://webscraping.ai API. It provides Chrome JS rendering, rotating proxies and HTML parsing for web scraping.
+
+The version of the OpenAPI document: 1.0.0
+
+Generated by: https://openapi-generator.tech
+OpenAPI Generator version: 4.2.3
+
+=end
+
+require 'cgi'
+
+module WebScrapingAI
+  class HtmlApi
+    attr_accessor :api_client
+
+    def initialize(api_client = ApiClient.default)
+      @api_client = api_client
+    end
+    # Get page HTML by URL (renders JS in Chrome and uses rotating proxies)
+    # @param url [String] URL of the page to get
+    # @param [Hash] opts the optional parameters
+    # @option opts [String] :selector CSS selector to get a part of the page (null by default, returns whole page HTML)
+    # @option opts [Boolean] :outer_html Return outer HTML of the selected element (false by default, returns inner HTML)
+    # @option opts [String] :proxy Proxy country code, for geotargeting (US by default)
+    # @option opts [Boolean] :disable_js Disable JS execution (false by default)
+    # @option opts [Boolean] :inline_css Inline included CSS files to make page viewable on other domains (false by default)
+    # @return [ScrappedPage]
+    def get_page(url, opts = {})
+      data, _status_code, _headers = get_page_with_http_info(url, opts)
+      data
+    end
+
+    # Get page HTML by URL (renders JS in Chrome and uses rotating proxies)
+    # @param url [String] URL of the page to get
+    # @param [Hash] opts the optional parameters
+    # @option opts [String] :selector CSS selector to get a part of the page (null by default, returns whole page HTML)
+    # @option opts [Boolean] :outer_html Return outer HTML of the selected element (false by default, returns inner HTML)
+    # @option opts [String] :proxy Proxy country code, for geotargeting (US by default)
+    # @option opts [Boolean] :disable_js Disable JS execution (false by default)
+    # @option opts [Boolean] :inline_css Inline included CSS files to make page viewable on other domains (false by default)
+    # @return [Array<(ScrappedPage, Integer, Hash)>] ScrappedPage data, response status code and response headers
+    def get_page_with_http_info(url, opts = {})
+      if @api_client.config.debugging
+        @api_client.config.logger.debug 'Calling API: HtmlApi.get_page ...'
+      end
+      # verify the required parameter 'url' is set
+      if @api_client.config.client_side_validation && url.nil?
+        fail ArgumentError, "Missing the required parameter 'url' when calling HtmlApi.get_page"
+      end
+      # resource path
+      local_var_path = '/'
+
+      # query parameters
+      query_params = opts[:query_params] || {}
+      query_params[:'url'] = url
+      query_params[:'selector'] = opts[:'selector'] if !opts[:'selector'].nil?
+      query_params[:'outer_html'] = opts[:'outer_html'] if !opts[:'outer_html'].nil?
+      query_params[:'proxy'] = opts[:'proxy'] if !opts[:'proxy'].nil?
+      query_params[:'disable_js'] = opts[:'disable_js'] if !opts[:'disable_js'].nil?
+      query_params[:'inline_css'] = opts[:'inline_css'] if !opts[:'inline_css'].nil?
+
+      # header parameters
+      header_params = opts[:header_params] || {}
+      # HTTP header 'Accept' (if needed)
+      header_params['Accept'] = @api_client.select_header_accept(['application/json'])
+
+      # form parameters
+      form_params = opts[:form_params] || {}
+
+      # http body (model)
+      post_body = opts[:body] 
+
+      # return_type
+      return_type = opts[:return_type] || 'ScrappedPage' 
+
+      # auth_names
+      auth_names = opts[:auth_names] || ['api_key']
+
+      new_options = opts.merge(
+        :header_params => header_params,
+        :query_params => query_params,
+        :form_params => form_params,
+        :body => post_body,
+        :auth_names => auth_names,
+        :return_type => return_type
+      )
+
+      data, status_code, headers = @api_client.call_api(:GET, local_var_path, new_options)
+      if @api_client.config.debugging
+        @api_client.config.logger.debug "API called: HtmlApi#get_page\nData: #{data.inspect}\nStatus code: #{status_code}\nHeaders: #{headers}"
+      end
+      return data, status_code, headers
+    end
+  end
+end