webscraping_ai 1.0.0

data/lib/webscraping_ai/models/scrapped_page.rb

@@ -0,0 +1,237 @@
+=begin
+#WebScraping.AI
+
+#A client for https://webscraping.ai API. It provides Chrome JS rendering, rotating proxies and HTML parsing for web scraping.
+
+The version of the OpenAPI document: 1.0.0
+
+Generated by: https://openapi-generator.tech
+OpenAPI Generator version: 4.2.3
+
+=end
+
+require 'date'
+
+module WebScrapingAI
+  class ScrappedPage
+    # Page HTML content size in bytes
+    attr_accessor :size_bytes
+
+    # HTML of the full page or a selected area
+    attr_accessor :html
+
+    # Response HTTP status code (200, 404, 302, etc)
+    attr_accessor :status
+
+    # Response HTTP status message
+    attr_accessor :status_message
+
+    # Attribute mapping from ruby-style variable name to JSON key.
+    def self.attribute_map
+      {
+        :'size_bytes' => :'size_bytes',
+        :'html' => :'html',
+        :'status' => :'status',
+        :'status_message' => :'statusMessage'
+      }
+    end
+
+    # Attribute type mapping.
+    def self.openapi_types
+      {
+        :'size_bytes' => :'Integer',
+        :'html' => :'String',
+        :'status' => :'Integer',
+        :'status_message' => :'String'
+      }
+    end
+
+    # List of attributes with nullable: true
+    def self.openapi_nullable
+      Set.new([
+      ])
+    end
+
+    # Initializes the object
+    # @param [Hash] attributes Model attributes in the form of hash
+    def initialize(attributes = {})
+      if (!attributes.is_a?(Hash))
+        fail ArgumentError, "The input argument (attributes) must be a hash in `WebScrapingAI::ScrappedPage` initialize method"
+      end
+
+      # check to see if the attribute exists and convert string to symbol for hash key
+      attributes = attributes.each_with_object({}) { |(k, v), h|
+        if (!self.class.attribute_map.key?(k.to_sym))
+          fail ArgumentError, "`#{k}` is not a valid attribute in `WebScrapingAI::ScrappedPage`. Please check the name to make sure it's valid. List of attributes: " + self.class.attribute_map.keys.inspect
+        end
+        h[k.to_sym] = v
+      }
+
+      if attributes.key?(:'size_bytes')
+        self.size_bytes = attributes[:'size_bytes']
+      end
+
+      if attributes.key?(:'html')
+        self.html = attributes[:'html']
+      end
+
+      if attributes.key?(:'status')
+        self.status = attributes[:'status']
+      end
+
+      if attributes.key?(:'status_message')
+        self.status_message = attributes[:'status_message']
+      end
+    end
+
+    # Show invalid properties with the reasons. Usually used together with valid?
+    # @return Array for valid properties with the reasons
+    def list_invalid_properties
+      invalid_properties = Array.new
+      invalid_properties
+    end
+
+    # Check to see if the all the properties in the model are valid
+    # @return true if the model is valid
+    def valid?
+      true
+    end
+
+    # Checks equality by comparing each attribute.
+    # @param [Object] Object to be compared
+    def ==(o)
+      return true if self.equal?(o)
+      self.class == o.class &&
+          size_bytes == o.size_bytes &&
+          html == o.html &&
+          status == o.status &&
+          status_message == o.status_message
+    end
+
+    # @see the `==` method
+    # @param [Object] Object to be compared
+    def eql?(o)
+      self == o
+    end
+
+    # Calculates hash code according to all attributes.
+    # @return [Integer] Hash code
+    def hash
+      [size_bytes, html, status, status_message].hash
+    end
+
+    # Builds the object from hash
+    # @param [Hash] attributes Model attributes in the form of hash
+    # @return [Object] Returns the model itself
+    def self.build_from_hash(attributes)
+      new.build_from_hash(attributes)
+    end
+
+    # Builds the object from hash
+    # @param [Hash] attributes Model attributes in the form of hash
+    # @return [Object] Returns the model itself
+    def build_from_hash(attributes)
+      return nil unless attributes.is_a?(Hash)
+      self.class.openapi_types.each_pair do |key, type|
+        if type =~ /\AArray<(.*)>/i
+          # check to ensure the input is an array given that the attribute
+          # is documented as an array but the input is not
+          if attributes[self.class.attribute_map[key]].is_a?(Array)
+            self.send("#{key}=", attributes[self.class.attribute_map[key]].map { |v| _deserialize($1, v) })
+          end
+        elsif !attributes[self.class.attribute_map[key]].nil?
+          self.send("#{key}=", _deserialize(type, attributes[self.class.attribute_map[key]]))
+        end # or else data not found in attributes(hash), not an issue as the data can be optional
+      end
+
+      self
+    end
+
+    # Deserializes the data based on type
+    # @param string type Data type
+    # @param string value Value to be deserialized
+    # @return [Object] Deserialized data
+    def _deserialize(type, value)
+      case type.to_sym
+      when :DateTime
+        DateTime.parse(value)
+      when :Date
+        Date.parse(value)
+      when :String
+        value.to_s
+      when :Integer
+        value.to_i
+      when :Float
+        value.to_f
+      when :Boolean
+        if value.to_s =~ /\A(true|t|yes|y|1)\z/i
+          true
+        else
+          false
+        end
+      when :Object
+        # generic object (usually a Hash), return directly
+        value
+      when /\AArray<(?<inner_type>.+)>\z/
+        inner_type = Regexp.last_match[:inner_type]
+        value.map { |v| _deserialize(inner_type, v) }
+      when /\AHash<(?<k_type>.+?), (?<v_type>.+)>\z/
+        k_type = Regexp.last_match[:k_type]
+        v_type = Regexp.last_match[:v_type]
+        {}.tap do |hash|
+          value.each do |k, v|
+            hash[_deserialize(k_type, k)] = _deserialize(v_type, v)
+          end
+        end
+      else # model
+        WebScrapingAI.const_get(type).build_from_hash(value)
+      end
+    end
+
+    # Returns the string representation of the object
+    # @return [String] String presentation of the object
+    def to_s
+      to_hash.to_s
+    end
+
+    # to_body is an alias to to_hash (backward compatibility)
+    # @return [Hash] Returns the object in the form of hash
+    def to_body
+      to_hash
+    end
+
+    # Returns the object in the form of hash
+    # @return [Hash] Returns the object in the form of hash
+    def to_hash
+      hash = {}
+      self.class.attribute_map.each_pair do |attr, param|
+        value = self.send(attr)
+        if value.nil?
+          is_nullable = self.class.openapi_nullable.include?(attr)
+          next if !is_nullable || (is_nullable && !instance_variable_defined?(:"@#{attr}"))
+        end
+        
+        hash[param] = _to_hash(value)
+      end
+      hash
+    end
+
+    # Outputs non-array value in the form of hash
+    # For object, use to_hash. Otherwise, just return the value
+    # @param [Object] value Any valid value
+    # @return [Hash] Returns the value in the form of hash
+    def _to_hash(value)
+      if value.is_a?(Array)
+        value.compact.map { |v| _to_hash(v) }
+      elsif value.is_a?(Hash)
+        {}.tap do |hash|
+          value.each { |k, v| hash[k] = _to_hash(v) }
+        end
+      elsif value.respond_to? :to_hash
+        value.to_hash
+      else
+        value
+      end
+    end
+  end
+end

data/lib/webscraping_ai/version.rb

@@ -0,0 +1,15 @@
+=begin
+#WebScraping.AI
+
+#A client for https://webscraping.ai API. It provides Chrome JS rendering, rotating proxies and HTML parsing for web scraping.
+
+The version of the OpenAPI document: 1.0.0
+
+Generated by: https://openapi-generator.tech
+OpenAPI Generator version: 4.2.3
+
+=end
+
+module WebScrapingAI
+  VERSION = '1.0.0'
+end

data/lib/webscraping_ai.rb

@@ -0,0 +1,41 @@
+=begin
+#WebScraping.AI
+
+#A client for https://webscraping.ai API. It provides Chrome JS rendering, rotating proxies and HTML parsing for web scraping.
+
+The version of the OpenAPI document: 1.0.0
+
+Generated by: https://openapi-generator.tech
+OpenAPI Generator version: 4.2.3
+
+=end
+
+# Common files
+require 'webscraping_ai/api_client'
+require 'webscraping_ai/api_error'
+require 'webscraping_ai/version'
+require 'webscraping_ai/configuration'
+
+# Models
+require 'webscraping_ai/models/scrapped_page'
+
+# APIs
+require 'webscraping_ai/api/html_api'
+
+module WebScrapingAI
+  class << self
+    # Customize default settings for the SDK using block.
+    #   WebScrapingAI.configure do |config|
+    #     config.username = "xxx"
+    #     config.password = "xxx"
+    #   end
+    # If no block given, return the default Configuration object.
+    def configure
+      if block_given?
+        yield(Configuration.default)
+      else
+        Configuration.default
+      end
+    end
+  end
+end

data/spec/api/html_api_spec.rb

@@ -0,0 +1,46 @@
+=begin
+#WebScraping.AI
+
+#This is a sample server Petstore server. For this sample, you can use the api key `special-key` to test the authorization filters.
+
+The version of the OpenAPI document: 1.0.0
+
+Generated by: https://openapi-generator.tech
+OpenAPI Generator version: 4.2.3
+
+=end
+
+require 'spec_helper'
+require 'json'
+
+# Unit tests for WebScrapingAI::HtmlApi
+# Automatically generated by openapi-generator (https://openapi-generator.tech)
+# Please update as you see appropriate
+describe 'HtmlApi' do
+  before do
+    # run before each test
+    @api_instance = WebScrapingAI::HtmlApi.new
+  end
+
+  after do
+    # run after each test
+  end
+
+  describe 'test an instance of HtmlApi' do
+    it 'should create an instance of HtmlApi' do
+      expect(@api_instance).to be_instance_of(WebScrapingAI::HtmlApi)
+    end
+  end
+
+  # unit tests for get_page
+  # Get page HTML by URL
+  # @param url URL of the page to get
+  # @param [Hash] opts the optional parameters
+  # @return [ScrappedPage]
+  describe 'get_page test' do
+    it 'should work' do
+      # assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
+    end
+  end
+
+end

data/spec/api_client_spec.rb

@@ -0,0 +1,226 @@
+=begin
+#WebScraping.AI
+
+#This is a sample server Petstore server. For this sample, you can use the api key `special-key` to test the authorization filters.
+
+The version of the OpenAPI document: 1.0.0
+
+Generated by: https://openapi-generator.tech
+OpenAPI Generator version: 4.2.3
+
+=end
+
+require 'spec_helper'
+
+describe WebScrapingAI::ApiClient do
+  context 'initialization' do
+    context 'URL stuff' do
+      context 'host' do
+        it 'removes http from host' do
+          WebScrapingAI.configure { |c| c.host = 'http://example.com' }
+          expect(WebScrapingAI::Configuration.default.host).to eq('example.com')
+        end
+
+        it 'removes https from host' do
+          WebScrapingAI.configure { |c| c.host = 'https://wookiee.com' }
+          expect(WebScrapingAI::ApiClient.default.config.host).to eq('wookiee.com')
+        end
+
+        it 'removes trailing path from host' do
+          WebScrapingAI.configure { |c| c.host = 'hobo.com/v4' }
+          expect(WebScrapingAI::Configuration.default.host).to eq('hobo.com')
+        end
+      end
+
+      context 'base_path' do
+        it "prepends a slash to base_path" do
+          WebScrapingAI.configure { |c| c.base_path = 'v4/dog' }
+          expect(WebScrapingAI::Configuration.default.base_path).to eq('/v4/dog')
+        end
+
+        it "doesn't prepend a slash if one is already there" do
+          WebScrapingAI.configure { |c| c.base_path = '/v4/dog' }
+          expect(WebScrapingAI::Configuration.default.base_path).to eq('/v4/dog')
+        end
+
+        it "ends up as a blank string if nil" do
+          WebScrapingAI.configure { |c| c.base_path = nil }
+          expect(WebScrapingAI::Configuration.default.base_path).to eq('')
+        end
+      end
+    end
+  end
+
+  describe 'params_encoding in #build_request' do
+    let(:config) { WebScrapingAI::Configuration.new }
+    let(:api_client) { WebScrapingAI::ApiClient.new(config) }
+
+    it 'defaults to nil' do
+      expect(WebScrapingAI::Configuration.default.params_encoding).to eq(nil)
+      expect(config.params_encoding).to eq(nil)
+
+      request = api_client.build_request(:get, '/test')
+      expect(request.options[:params_encoding]).to eq(nil)
+    end
+
+    it 'can be customized' do
+      config.params_encoding = :multi
+      request = api_client.build_request(:get, '/test')
+      expect(request.options[:params_encoding]).to eq(:multi)
+    end
+  end
+
+  describe 'timeout in #build_request' do
+    let(:config) { WebScrapingAI::Configuration.new }
+    let(:api_client) { WebScrapingAI::ApiClient.new(config) }
+
+    it 'defaults to 0' do
+      expect(WebScrapingAI::Configuration.default.timeout).to eq(0)
+      expect(config.timeout).to eq(0)
+
+      request = api_client.build_request(:get, '/test')
+      expect(request.options[:timeout]).to eq(0)
+    end
+
+    it 'can be customized' do
+      config.timeout = 100
+      request = api_client.build_request(:get, '/test')
+      expect(request.options[:timeout]).to eq(100)
+    end
+  end
+
+  describe '#deserialize' do
+    it "handles Array<Integer>" do
+      api_client = WebScrapingAI::ApiClient.new
+      headers = { 'Content-Type' => 'application/json' }
+      response = double('response', headers: headers, body: '[12, 34]')
+      data = api_client.deserialize(response, 'Array<Integer>')
+      expect(data).to be_instance_of(Array)
+      expect(data).to eq([12, 34])
+    end
+
+    it 'handles Array<Array<Integer>>' do
+      api_client = WebScrapingAI::ApiClient.new
+      headers = { 'Content-Type' => 'application/json' }
+      response = double('response', headers: headers, body: '[[12, 34], [56]]')
+      data = api_client.deserialize(response, 'Array<Array<Integer>>')
+      expect(data).to be_instance_of(Array)
+      expect(data).to eq([[12, 34], [56]])
+    end
+
+    it 'handles Hash<String, String>' do
+      api_client = WebScrapingAI::ApiClient.new
+      headers = { 'Content-Type' => 'application/json' }
+      response = double('response', headers: headers, body: '{"message": "Hello"}')
+      data = api_client.deserialize(response, 'Hash<String, String>')
+      expect(data).to be_instance_of(Hash)
+      expect(data).to eq(:message => 'Hello')
+    end
+  end
+
+  describe "#object_to_hash" do
+    it 'ignores nils and includes empty arrays' do
+      # uncomment below to test object_to_hash for model
+      # api_client = WebScrapingAI::ApiClient.new
+      # _model = WebScrapingAI::ModelName.new
+      # update the model attribute below
+      # _model.id = 1
+      # update the expected value (hash) below
+      # expected = {id: 1, name: '', tags: []}
+      # expect(api_client.object_to_hash(_model)).to eq(expected)
+    end
+  end
+
+  describe '#build_collection_param' do
+    let(:param) { ['aa', 'bb', 'cc'] }
+    let(:api_client) { WebScrapingAI::ApiClient.new }
+
+    it 'works for csv' do
+      expect(api_client.build_collection_param(param, :csv)).to eq('aa,bb,cc')
+    end
+
+    it 'works for ssv' do
+      expect(api_client.build_collection_param(param, :ssv)).to eq('aa bb cc')
+    end
+
+    it 'works for tsv' do
+      expect(api_client.build_collection_param(param, :tsv)).to eq("aa\tbb\tcc")
+    end
+
+    it 'works for pipes' do
+      expect(api_client.build_collection_param(param, :pipes)).to eq('aa|bb|cc')
+    end
+
+    it 'works for multi' do
+      expect(api_client.build_collection_param(param, :multi)).to eq(['aa', 'bb', 'cc'])
+    end
+
+    it 'fails for invalid collection format' do
+      expect{api_client.build_collection_param(param, :INVALID)}.to raise_error(RuntimeError, 'unknown collection format: :INVALID')
+    end
+  end
+
+  describe '#json_mime?' do
+    let(:api_client) { WebScrapingAI::ApiClient.new }
+
+    it 'works' do
+      expect(api_client.json_mime?(nil)).to eq false
+      expect(api_client.json_mime?('')).to eq false
+
+      expect(api_client.json_mime?('application/json')).to eq true
+      expect(api_client.json_mime?('application/json; charset=UTF8')).to eq true
+      expect(api_client.json_mime?('APPLICATION/JSON')).to eq true
+
+      expect(api_client.json_mime?('application/xml')).to eq false
+      expect(api_client.json_mime?('text/plain')).to eq false
+      expect(api_client.json_mime?('application/jsonp')).to eq false
+    end
+  end
+
+  describe '#select_header_accept' do
+    let(:api_client) { WebScrapingAI::ApiClient.new }
+
+    it 'works' do
+      expect(api_client.select_header_accept(nil)).to be_nil
+      expect(api_client.select_header_accept([])).to be_nil
+
+      expect(api_client.select_header_accept(['application/json'])).to eq('application/json')
+      expect(api_client.select_header_accept(['application/xml', 'application/json; charset=UTF8'])).to eq('application/json; charset=UTF8')
+      expect(api_client.select_header_accept(['APPLICATION/JSON', 'text/html'])).to eq('APPLICATION/JSON')
+
+      expect(api_client.select_header_accept(['application/xml'])).to eq('application/xml')
+      expect(api_client.select_header_accept(['text/html', 'application/xml'])).to eq('text/html,application/xml')
+    end
+  end
+
+  describe '#select_header_content_type' do
+    let(:api_client) { WebScrapingAI::ApiClient.new }
+
+    it 'works' do
+      expect(api_client.select_header_content_type(nil)).to eq('application/json')
+      expect(api_client.select_header_content_type([])).to eq('application/json')
+
+      expect(api_client.select_header_content_type(['application/json'])).to eq('application/json')
+      expect(api_client.select_header_content_type(['application/xml', 'application/json; charset=UTF8'])).to eq('application/json; charset=UTF8')
+      expect(api_client.select_header_content_type(['APPLICATION/JSON', 'text/html'])).to eq('APPLICATION/JSON')
+      expect(api_client.select_header_content_type(['application/xml'])).to eq('application/xml')
+      expect(api_client.select_header_content_type(['text/plain', 'application/xml'])).to eq('text/plain')
+    end
+  end
+
+  describe '#sanitize_filename' do
+    let(:api_client) { WebScrapingAI::ApiClient.new }
+
+    it 'works' do
+      expect(api_client.sanitize_filename('sun')).to eq('sun')
+      expect(api_client.sanitize_filename('sun.gif')).to eq('sun.gif')
+      expect(api_client.sanitize_filename('../sun.gif')).to eq('sun.gif')
+      expect(api_client.sanitize_filename('/var/tmp/sun.gif')).to eq('sun.gif')
+      expect(api_client.sanitize_filename('./sun.gif')).to eq('sun.gif')
+      expect(api_client.sanitize_filename('..\sun.gif')).to eq('sun.gif')
+      expect(api_client.sanitize_filename('\var\tmp\sun.gif')).to eq('sun.gif')
+      expect(api_client.sanitize_filename('c:\var\tmp\sun.gif')).to eq('sun.gif')
+      expect(api_client.sanitize_filename('.\sun.gif')).to eq('sun.gif')
+    end
+  end
+end

data/spec/configuration_spec.rb

@@ -0,0 +1,42 @@
+=begin
+#WebScraping.AI
+
+#This is a sample server Petstore server. For this sample, you can use the api key `special-key` to test the authorization filters.
+
+The version of the OpenAPI document: 1.0.0
+
+Generated by: https://openapi-generator.tech
+OpenAPI Generator version: 4.2.3
+
+=end
+
+require 'spec_helper'
+
+describe WebScrapingAI::Configuration do
+  let(:config) { WebScrapingAI::Configuration.default }
+
+  before(:each) do
+    # uncomment below to setup host and base_path
+    # require 'URI'
+    # uri = URI.parse("https://webscraping.ai/api")
+    # WebScrapingAI.configure do |c|
+    #   c.host = uri.host
+    #   c.base_path = uri.path
+    # end
+  end
+
+  describe '#base_url' do
+    it 'should have the default value' do
+      # uncomment below to test default value of the base path
+      # expect(config.base_url).to eq("https://webscraping.ai/api")
+    end
+
+    it 'should remove trailing slashes' do
+      [nil, '', '/', '//'].each do |base_path|
+        config.base_path = base_path
+        # uncomment below to test trailing slashes
+        # expect(config.base_url).to eq("https://webscraping.ai/api")
+      end
+    end
+  end
+end

data/spec/models/scrapped_page_spec.rb

@@ -0,0 +1,59 @@
+=begin
+#WebScraping.AI
+
+#This is a sample server Petstore server. For this sample, you can use the api key `special-key` to test the authorization filters.
+
+The version of the OpenAPI document: 1.0.0
+
+Generated by: https://openapi-generator.tech
+OpenAPI Generator version: 4.2.3
+
+=end
+
+require 'spec_helper'
+require 'json'
+require 'date'
+
+# Unit tests for WebScrapingAI::ScrappedPage
+# Automatically generated by openapi-generator (https://openapi-generator.tech)
+# Please update as you see appropriate
+describe 'ScrappedPage' do
+  before do
+    # run before each test
+    @instance = WebScrapingAI::ScrappedPage.new
+  end
+
+  after do
+    # run after each test
+  end
+
+  describe 'test an instance of ScrappedPage' do
+    it 'should create an instance of ScrappedPage' do
+      expect(@instance).to be_instance_of(WebScrapingAI::ScrappedPage)
+    end
+  end
+  describe 'test attribute "size_bytes"' do
+    it 'should work' do
+      # assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
+    end
+  end
+
+  describe 'test attribute "html"' do
+    it 'should work' do
+      # assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
+    end
+  end
+
+  describe 'test attribute "status"' do
+    it 'should work' do
+      # assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
+    end
+  end
+
+  describe 'test attribute "status_message"' do
+    it 'should work' do
+      # assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
+    end
+  end
+
+end