webscraping_ai 2.0.2 → 3.1.2

data/lib/webscraping_ai/api_client.rb

@@ -1,12 +1,12 @@
 =begin
 #WebScraping.AI
 
-#A client for https://webscraping.ai API. It provides a web scaping automation API with Chrome JS rendering, rotating proxies and builtin HTML parsing.
+#WebScraping.AI scraping API provides GPT-powered tools with Chromium JavaScript rendering, rotating proxies, and built-in HTML parsing.
 
-The version of the OpenAPI document: 2.0.2
+The version of the OpenAPI document: 3.1.2
 Contact: support@webscraping.ai
 Generated by: https://openapi-generator.tech
-OpenAPI Generator version: 4.3.1
+OpenAPI Generator version: 7.2.0
 
 =end
 
@@ -14,8 +14,10 @@ require 'date'
 require 'json'
 require 'logger'
 require 'tempfile'
+require 'time'
 require 'typhoeus'
 
+
 module WebScrapingAI
   class ApiClient
     # The Configuration object holding settings to be used in the API client.
@@ -44,9 +46,10 @@ module WebScrapingAI
     # Call an API with given options.
     #
     # @return [Array<(Object, Integer, Hash)>] an array of 3 elements:
-    #   the data deserialized from response body (could be nil), response status code and response headers.
+    #   the data deserialized from response body (may be a Tempfile or nil), response status code and response headers.
     def call_api(http_method, path, opts = {})
       request = build_request(http_method, path, opts)
+      tempfile = download_file(request) if opts[:return_type] == 'File'
       response = request.run
 
       if @config.debugging
@@ -68,7 +71,9 @@ module WebScrapingAI
         end
       end
 
-      if opts[:return_type]
+      if opts[:return_type] == 'File'
+        data = tempfile
+      elsif opts[:return_type]
         data = deserialize(response, opts[:return_type])
       else
         data = nil
@@ -86,12 +91,13 @@ module WebScrapingAI
     # @option opts [Object] :body HTTP body (JSON/XML)
     # @return [Typhoeus::Request] A Typhoeus Request
     def build_request(http_method, path, opts = {})
-      url = build_request_url(path)
+      url = build_request_url(path, opts)
       http_method = http_method.to_sym.downcase
 
       header_params = @default_headers.merge(opts[:header_params] || {})
       query_params = opts[:query_params] || {}
       form_params = opts[:form_params] || {}
+      follow_location = opts[:follow_location] || true
 
       update_params_for_auth! header_params, query_params, opts[:auth_names]
 
@@ -108,7 +114,8 @@ module WebScrapingAI
         :ssl_verifyhost => _verify_ssl_host,
         :sslcert => @config.cert_file,
         :sslkey => @config.key_file,
-        :verbose => @config.debugging
+        :verbose => @config.debugging,
+        :followlocation => follow_location
       }
 
       # set custom cert, if provided
@@ -122,9 +129,7 @@ module WebScrapingAI
         end
       end
 
-      request = Typhoeus::Request.new(url, req_opts)
-      download_file(request) if opts[:return_type] == 'File'
-      request
+      Typhoeus::Request.new(url, req_opts)
     end
 
     # Builds the HTTP request body
@@ -155,6 +160,49 @@ module WebScrapingAI
       data
     end
 
+    # Save response body into a file in (the defined) temporary folder, using the filename
+    # from the "Content-Disposition" header if provided, otherwise a random filename.
+    # The response body is written to the file in chunks in order to handle files which
+    # size is larger than maximum Ruby String or even larger than the maximum memory a Ruby
+    # process can use.
+    #
+    # @see Configuration#temp_folder_path
+    #
+    # @return [Tempfile] the tempfile generated
+    def download_file(request)
+      tempfile = nil
+      encoding = nil
+      request.on_headers do |response|
+        content_disposition = response.headers['Content-Disposition']
+        if content_disposition && content_disposition =~ /filename=/i
+          filename = content_disposition[/filename=['"]?([^'"\s]+)['"]?/, 1]
+          prefix = sanitize_filename(filename)
+        else
+          prefix = 'download-'
+        end
+        prefix = prefix + '-' unless prefix.end_with?('-')
+        encoding = response.body.encoding
+        tempfile = Tempfile.open(prefix, @config.temp_folder_path, encoding: encoding)
+      end
+      request.on_body do |chunk|
+        chunk.force_encoding(encoding)
+        tempfile.write(chunk)
+      end
+      # run the request to ensure the tempfile is created successfully before returning it
+      request.run
+      if tempfile
+        tempfile.close
+        @config.logger.info "Temp file written to #{tempfile.path}, please copy the file to a proper folder "\
+                            "with e.g. `FileUtils.cp(tempfile.path, '/new/file/path')` otherwise the temp file "\
+                            "will be deleted automatically with GC. It's also recommended to delete the temp file "\
+                            "explicitly with `tempfile.delete`"
+      else
+        fail ApiError.new("Failed to create the tempfile based on the HTTP response from the server: #{request.inspect}") 
+      end
+
+      tempfile
+    end
+
     # Check if the given MIME is a JSON MIME.
     # JSON MIME examples:
     #   application/json
@@ -173,15 +221,10 @@ module WebScrapingAI
     # @param [String] return_type some examples: "User", "Array<User>", "Hash<String, Integer>"
     def deserialize(response, return_type)
       body = response.body
-
-      # handle file downloading - return the File instance processed in request callbacks
-      # note that response body is empty when the file is written in chunks in request on_body callback
-      return @tempfile if return_type == 'File'
-
       return nil if body.nil? || body.empty?
 
       # return response body directly for String return type
-      return body if return_type == 'String'
+      return body.to_s if return_type == 'String'
 
       # ensuring a default content type
       content_type = response.headers['Content-Type'] || 'application/json'
@@ -191,7 +234,7 @@ module WebScrapingAI
       begin
         data = JSON.parse("[#{body}]", :symbolize_names => true)[0]
       rescue JSON::ParserError => e
-        if %w(String Date DateTime).include?(return_type)
+        if %w(String Date Time).include?(return_type)
           data = body
         else
           raise e
@@ -216,9 +259,9 @@ module WebScrapingAI
         data.to_f
       when 'Boolean'
         data == true
-      when 'DateTime'
+      when 'Time'
         # parse date time (expecting ISO 8601 format)
-        DateTime.parse data
+        Time.parse data
       when 'Date'
         # parse date time (expecting ISO 8601 format)
         Date.parse data
@@ -236,46 +279,9 @@ module WebScrapingAI
           data.each { |k, v| hash[k] = convert_to_type(v, sub_type) }
         end
       else
-        # models, e.g. Pet
-        WebScrapingAI.const_get(return_type).build_from_hash(data)
-      end
-    end
-
-    # Save response body into a file in (the defined) temporary folder, using the filename
-    # from the "Content-Disposition" header if provided, otherwise a random filename.
-    # The response body is written to the file in chunks in order to handle files which
-    # size is larger than maximum Ruby String or even larger than the maximum memory a Ruby
-    # process can use.
-    #
-    # @see Configuration#temp_folder_path
-    def download_file(request)
-      tempfile = nil
-      encoding = nil
-      request.on_headers do |response|
-        content_disposition = response.headers['Content-Disposition']
-        if content_disposition && content_disposition =~ /filename=/i
-          filename = content_disposition[/filename=['"]?([^'"\s]+)['"]?/, 1]
-          prefix = sanitize_filename(filename)
-        else
-          prefix = 'download-'
-        end
-        prefix = prefix + '-' unless prefix.end_with?('-')
-        encoding = response.body.encoding
-        tempfile = Tempfile.open(prefix, @config.temp_folder_path, encoding: encoding)
-        @tempfile = tempfile
-      end
-      request.on_body do |chunk|
-        chunk.force_encoding(encoding)
-        tempfile.write(chunk)
-      end
-      request.on_complete do |response|
-        if tempfile
-          tempfile.close
-          @config.logger.info "Temp file written to #{tempfile.path}, please copy the file to a proper folder "\
-                              "with e.g. `FileUtils.cp(tempfile.path, '/new/file/path')` otherwise the temp file "\
-                              "will be deleted automatically with GC. It's also recommended to delete the temp file "\
-                              "explicitly with `tempfile.delete`"
-        end
+        # models (e.g. Pet) or oneOf
+        klass = WebScrapingAI.const_get(return_type)
+        klass.respond_to?(:openapi_one_of) ? klass.build(data) : klass.build_from_hash(data)
       end
     end
 
@@ -288,13 +294,13 @@ module WebScrapingAI
       filename.gsub(/.*[\/\\]/, '')
     end
 
-    def build_request_url(path)
+    def build_request_url(path, opts = {})
       # Add leading and trailing slashes to path
       path = "/#{path}".gsub(/\/+/, '/')
-      @config.base_url + path
+      @config.base_url(opts[:operation]) + path
     end
 
-    # Update hearder and query params based on authentication settings.
+    # Update header and query params based on authentication settings.
     #
     # @param [Hash] header_params Header parameters
     # @param [Hash] query_params Query parameters
@@ -306,7 +312,7 @@ module WebScrapingAI
         case auth_setting[:in]
         when 'header' then header_params[auth_setting[:key]] = auth_setting[:value]
         when 'query'  then query_params[auth_setting[:key]] = auth_setting[:value]
-        else fail ArgumentError, 'Authentication token must be in `query` of `header`'
+        else fail ArgumentError, 'Authentication token must be in `query` or `header`'
         end
       end
     end
@@ -333,8 +339,8 @@ module WebScrapingAI
     # @param [Array] content_types array for Content-Type
     # @return [String] the Content-Type header  (e.g. application/json)
     def select_header_content_type(content_types)
-      # use application/json by default
-      return 'application/json' if content_types.nil? || content_types.empty?
+      # return nil by default
+      return if content_types.nil? || content_types.empty?
       # use JSON when present, otherwise use the first one
       json_content_type = content_types.find { |s| json_mime?(s) }
       json_content_type || content_types.first

data/lib/webscraping_ai/api_error.rb

@@ -1,12 +1,12 @@
 =begin
 #WebScraping.AI
 
-#A client for https://webscraping.ai API. It provides a web scaping automation API with Chrome JS rendering, rotating proxies and builtin HTML parsing.
+#WebScraping.AI scraping API provides GPT-powered tools with Chromium JavaScript rendering, rotating proxies, and built-in HTML parsing.
 
-The version of the OpenAPI document: 2.0.2
+The version of the OpenAPI document: 3.1.2
 Contact: support@webscraping.ai
 Generated by: https://openapi-generator.tech
-OpenAPI Generator version: 4.3.1
+OpenAPI Generator version: 7.2.0
 
 =end
 
@@ -32,6 +32,7 @@ module WebScrapingAI
         end
       else
         super arg
+        @message = arg
       end
     end
 

data/lib/webscraping_ai/configuration.rb

@@ -1,12 +1,12 @@
 =begin
 #WebScraping.AI
 
-#A client for https://webscraping.ai API. It provides a web scaping automation API with Chrome JS rendering, rotating proxies and builtin HTML parsing.
+#WebScraping.AI scraping API provides GPT-powered tools with Chromium JavaScript rendering, rotating proxies, and built-in HTML parsing.
 
-The version of the OpenAPI document: 2.0.2
+The version of the OpenAPI document: 3.1.2
 Contact: support@webscraping.ai
 Generated by: https://openapi-generator.tech
-OpenAPI Generator version: 4.3.1
+OpenAPI Generator version: 7.2.0
 
 =end
 
@@ -21,6 +21,18 @@ module WebScrapingAI
     # Defines url base path
     attr_accessor :base_path
 
+    # Define server configuration index
+    attr_accessor :server_index
+
+    # Define server operation configuration index
+    attr_accessor :server_operation_index
+
+    # Default server variables
+    attr_accessor :server_variables
+
+    # Default server operation variables
+    attr_accessor :server_operation_variables
+
     # Defines API keys used with API Key authentications.
     #
     # @return [Hash] key: parameter name, value: parameter value (API key)
@@ -50,6 +62,16 @@ module WebScrapingAI
     # Defines the access token (Bearer) used with OAuth2.
     attr_accessor :access_token
 
+    # Defines a Proc used to fetch or refresh access tokens (Bearer) used with OAuth2.
+    # Overrides the access_token if set
+    # @return [Proc]
+    attr_accessor :access_token_getter
+
+    # Set this to return data as binary instead of downloading a temp file. When enabled (set to true)
+    # HTTP responses with return type `File` will be returned as a stream of binary data.
+    # Default to false.
+    attr_accessor :return_binary_data
+
     # Set this to enable/disable debugging. When enabled (set to true), HTTP request/response
     # details will be logged with `logger.debug` (see the `logger` attribute).
     # Default to false.
@@ -121,6 +143,7 @@ module WebScrapingAI
     # https://github.com/typhoeus/ethon/blob/master/lib/ethon/easy/queryable.rb#L96
     attr_accessor :params_encoding
 
+
     attr_accessor :inject_format
 
     attr_accessor :force_ending_format
@@ -129,15 +152,19 @@ module WebScrapingAI
       @scheme = 'https'
       @host = 'api.webscraping.ai'
       @base_path = ''
+      @server_index = nil
+      @server_operation_index = {}
+      @server_variables = {}
+      @server_operation_variables = {}
       @api_key = {}
       @api_key_prefix = {}
-      @timeout = 0
       @client_side_validation = true
       @verify_ssl = true
       @verify_ssl_host = true
-      @params_encoding = nil
       @cert_file = nil
       @key_file = nil
+      @timeout = 0
+      @params_encoding = nil
       @debugging = false
       @inject_format = false
       @force_ending_format = false
@@ -171,20 +198,34 @@ module WebScrapingAI
       @base_path = '' if @base_path == '/'
     end
 
-    def base_url
-      "#{scheme}://#{[host, base_path].join('/').gsub(/\/+/, '/')}".sub(/\/+\z/, '')
+    # Returns base URL for specified operation based on server settings
+    def base_url(operation = nil)
+      if operation_server_settings.key?(operation) then
+        index = server_operation_index.fetch(operation, server_index)
+        server_url(index.nil? ? 0 : index, server_operation_variables.fetch(operation, server_variables), operation_server_settings[operation])
+      else
+        server_index.nil? ? "#{scheme}://#{[host, base_path].join('/').gsub(/\/+/, '/')}".sub(/\/+\z/, '') : server_url(server_index, server_variables, nil)
+      end
     end
 
     # Gets API key (with prefix if set).
     # @param [String] param_name the parameter name of API key auth
-    def api_key_with_prefix(param_name)
+    def api_key_with_prefix(param_name, param_alias = nil)
+      key = @api_key[param_name]
+      key = @api_key.fetch(param_alias, key) unless param_alias.nil?
       if @api_key_prefix[param_name]
-        "#{@api_key_prefix[param_name]} #{@api_key[param_name]}"
+        "#{@api_key_prefix[param_name]} #{key}"
       else
-        @api_key[param_name]
+        key
       end
     end
 
+    # Gets access_token using access_token_getter or uses the static access_token
+    def access_token_with_refresh
+      return access_token if access_token_getter.nil?
+      access_token_getter.call
+    end
+
     # Gets Basic Auth token string
     def basic_auth_token
       'Basic ' + ["#{username}:#{password}"].pack('m').delete("\r\n")
@@ -213,25 +254,32 @@ module WebScrapingAI
       ]
     end
 
+    def operation_server_settings
+      {
+      }
+    end
+
     # Returns URL based on server settings
     #
     # @param index array index of the server settings
     # @param variables hash of variable and the corresponding value
-    def server_url(index, variables = {})
-      servers = server_settings
+    def server_url(index, variables = {}, servers = nil)
+      servers = server_settings if servers == nil
 
       # check array index out of bound
-      if (index < 0 || index >= servers.size)
-        fail ArgumentError, "Invalid index #{index} when selecting the server. Must be less than #{servers.size}"
+      if (index.nil? || index < 0 || index >= servers.size)
+        fail ArgumentError, "Invalid index #{index} when selecting the server. Must not be nil and must be less than #{servers.size}"
       end
 
       server = servers[index]
       url = server[:url]
 
+      return url unless server.key? :variables
+
       # go through variable and assign a value
       server[:variables].each do |name, variable|
         if variables.key?(name)
-          if (server[:variables][name][:enum_values].include? variables[name])
+          if (!server[:variables][name].key?(:enum_values) || server[:variables][name][:enum_values].include?(variables[name]))
             url.gsub! "{" + name.to_s + "}", variables[name]
           else
             fail ArgumentError, "The variable `#{name}` in the server URL has invalid value #{variables[name]}. Must be #{server[:variables][name][:enum_values]}."
@@ -244,5 +292,7 @@ module WebScrapingAI
 
       url
     end
+
+
   end
 end

data/lib/webscraping_ai/models/{page_error.rb → account.rb}

@@ -1,38 +1,49 @@
 =begin
 #WebScraping.AI
 
-#A client for https://webscraping.ai API. It provides a web scaping automation API with Chrome JS rendering, rotating proxies and builtin HTML parsing.
+#WebScraping.AI scraping API provides GPT-powered tools with Chromium JavaScript rendering, rotating proxies, and built-in HTML parsing.
 
-The version of the OpenAPI document: 2.0.2
+The version of the OpenAPI document: 3.1.2
 Contact: support@webscraping.ai
 Generated by: https://openapi-generator.tech
-OpenAPI Generator version: 4.3.1
+OpenAPI Generator version: 7.2.0
 
 =end
 
 require 'date'
+require 'time'
 
 module WebScrapingAI
-  class PageError
-    # Response HTTP status code (403, 500, etc)
-    attr_accessor :status_code
+  class Account
+    # Remaining API credits quota
+    attr_accessor :remaining_api_calls
 
-    # Response HTTP status message
-    attr_accessor :status_message
+    # Next billing cycle start time (UNIX timestamp)
+    attr_accessor :resets_at
+
+    # Remaining concurrent requests
+    attr_accessor :remaining_concurrency
 
     # Attribute mapping from ruby-style variable name to JSON key.
     def self.attribute_map
       {
-        :'status_code' => :'status_code',
-        :'status_message' => :'status_message'
+        :'remaining_api_calls' => :'remaining_api_calls',
+        :'resets_at' => :'resets_at',
+        :'remaining_concurrency' => :'remaining_concurrency'
       }
     end
 
+    # Returns all the JSON keys this model knows about
+    def self.acceptable_attributes
+      attribute_map.values
+    end
+
     # Attribute type mapping.
     def self.openapi_types
       {
-        :'status_code' => :'Integer',
-        :'status_message' => :'String'
+        :'remaining_api_calls' => :'Integer',
+        :'resets_at' => :'Integer',
+        :'remaining_concurrency' => :'Integer'
       }
     end
 
@@ -46,29 +57,34 @@ module WebScrapingAI
     # @param [Hash] attributes Model attributes in the form of hash
     def initialize(attributes = {})
       if (!attributes.is_a?(Hash))
-        fail ArgumentError, "The input argument (attributes) must be a hash in `WebScrapingAI::PageError` initialize method"
+        fail ArgumentError, "The input argument (attributes) must be a hash in `WebScrapingAI::Account` initialize method"
       end
 
       # check to see if the attribute exists and convert string to symbol for hash key
       attributes = attributes.each_with_object({}) { |(k, v), h|
         if (!self.class.attribute_map.key?(k.to_sym))
-          fail ArgumentError, "`#{k}` is not a valid attribute in `WebScrapingAI::PageError`. Please check the name to make sure it's valid. List of attributes: " + self.class.attribute_map.keys.inspect
+          fail ArgumentError, "`#{k}` is not a valid attribute in `WebScrapingAI::Account`. Please check the name to make sure it's valid. List of attributes: " + self.class.attribute_map.keys.inspect
         end
         h[k.to_sym] = v
       }
 
-      if attributes.key?(:'status_code')
-        self.status_code = attributes[:'status_code']
+      if attributes.key?(:'remaining_api_calls')
+        self.remaining_api_calls = attributes[:'remaining_api_calls']
+      end
+
+      if attributes.key?(:'resets_at')
+        self.resets_at = attributes[:'resets_at']
       end
 
-      if attributes.key?(:'status_message')
-        self.status_message = attributes[:'status_message']
+      if attributes.key?(:'remaining_concurrency')
+        self.remaining_concurrency = attributes[:'remaining_concurrency']
       end
     end
 
     # Show invalid properties with the reasons. Usually used together with valid?
     # @return Array for valid properties with the reasons
     def list_invalid_properties
+      warn '[DEPRECATED] the `list_invalid_properties` method is obsolete'
       invalid_properties = Array.new
       invalid_properties
     end
@@ -76,6 +92,7 @@ module WebScrapingAI
     # Check to see if the all the properties in the model are valid
     # @return true if the model is valid
     def valid?
+      warn '[DEPRECATED] the `valid?` method is obsolete'
       true
     end
 
@@ -84,8 +101,9 @@ module WebScrapingAI
     def ==(o)
       return true if self.equal?(o)
       self.class == o.class &&
-          status_code == o.status_code &&
-          status_message == o.status_message
+          remaining_api_calls == o.remaining_api_calls &&
+          resets_at == o.resets_at &&
+          remaining_concurrency == o.remaining_concurrency
     end
 
     # @see the `==` method
@@ -97,44 +115,40 @@ module WebScrapingAI
     # Calculates hash code according to all attributes.
     # @return [Integer] Hash code
     def hash
-      [status_code, status_message].hash
+      [remaining_api_calls, resets_at, remaining_concurrency].hash
     end
 
     # Builds the object from hash
     # @param [Hash] attributes Model attributes in the form of hash
     # @return [Object] Returns the model itself
     def self.build_from_hash(attributes)
-      new.build_from_hash(attributes)
-    end
-
-    # Builds the object from hash
-    # @param [Hash] attributes Model attributes in the form of hash
-    # @return [Object] Returns the model itself
-    def build_from_hash(attributes)
       return nil unless attributes.is_a?(Hash)
-      self.class.openapi_types.each_pair do |key, type|
-        if type =~ /\AArray<(.*)>/i
+      attributes = attributes.transform_keys(&:to_sym)
+      transformed_hash = {}
+      openapi_types.each_pair do |key, type|
+        if attributes.key?(attribute_map[key]) && attributes[attribute_map[key]].nil?
+          transformed_hash["#{key}"] = nil
+        elsif type =~ /\AArray<(.*)>/i
           # check to ensure the input is an array given that the attribute
           # is documented as an array but the input is not
-          if attributes[self.class.attribute_map[key]].is_a?(Array)
-            self.send("#{key}=", attributes[self.class.attribute_map[key]].map { |v| _deserialize($1, v) })
+          if attributes[attribute_map[key]].is_a?(Array)
+            transformed_hash["#{key}"] = attributes[attribute_map[key]].map { |v| _deserialize($1, v) }
           end
-        elsif !attributes[self.class.attribute_map[key]].nil?
-          self.send("#{key}=", _deserialize(type, attributes[self.class.attribute_map[key]]))
-        end # or else data not found in attributes(hash), not an issue as the data can be optional
+        elsif !attributes[attribute_map[key]].nil?
+          transformed_hash["#{key}"] = _deserialize(type, attributes[attribute_map[key]])
+        end
       end
-
-      self
+      new(transformed_hash)
     end
 
     # Deserializes the data based on type
     # @param string type Data type
     # @param string value Value to be deserialized
     # @return [Object] Deserialized data
-    def _deserialize(type, value)
+    def self._deserialize(type, value)
       case type.to_sym
-      when :DateTime
-        DateTime.parse(value)
+      when :Time
+        Time.parse(value)
       when :Date
         Date.parse(value)
       when :String
@@ -164,7 +178,9 @@ module WebScrapingAI
           end
         end
       else # model
-        WebScrapingAI.const_get(type).build_from_hash(value)
+        # models (e.g. Pet) or oneOf
+        klass = WebScrapingAI.const_get(type)
+        klass.respond_to?(:openapi_any_of) || klass.respond_to?(:openapi_one_of) ? klass.build(value) : klass.build_from_hash(value)
       end
     end
 
@@ -190,7 +206,7 @@ module WebScrapingAI
           is_nullable = self.class.openapi_nullable.include?(attr)
           next if !is_nullable || (is_nullable && !instance_variable_defined?(:"@#{attr}"))
         end
-        
+
         hash[param] = _to_hash(value)
       end
       hash
@@ -213,5 +229,7 @@ module WebScrapingAI
         value
       end
     end
+
   end
+
 end