tiny_grabber 0.2.8 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 51f1a586c038410a555be4a5d046134f92786f75
-  data.tar.gz: 0bdb153926a9553276d06f7c40666d366fcbf0a6
+  metadata.gz: 96bdccdeb24ccdbbb99e4cb2e73bf1450aca4e4f
+  data.tar.gz: 785bed54953a6faa41325aa8d8dda56f688b638d
 SHA512:
-  metadata.gz: d9e986b006f6734e043ddedd168bbc66925ed97380b18e1d1651f0dfedcd9c36caccccd13eb3fb363d5b292ada62c2c41209cd862649552cfaf175c7bb732b6e
-  data.tar.gz: a849cf33106b8a119da43e4bd05dbc3e1cd592ab78947e7d8868d2cc2c2b7e4043dcd03678c56ed12ae91e98b4a26e746f4cd51803ab6cd3c3d8c926433ac0c7
+  metadata.gz: 4256e04d8e42b404c09e32ad66bd99c4c1f38680be021fcb6e36e6e8a1e86e65be548945d853583fad0177d53e2732a275ec804079531757d477af91bd403674
+  data.tar.gz: 46f84bf20a8c0e4de69e3bea855c5d3a26bd66332c75fbb7e2e6ed1805364c05262e10d897f4ea458931ac842819baee55f0a3c4cd7ed731ce7207c2bdd8d149

data/Gemfile

@@ -2,3 +2,7 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in tiny_grabber.gemspec
 gemspec
+
+gem 'rubocop', require: false
+gem 'byebug'
+gem 'timezone', '~> 1.0'

data/README.md

@@ -126,6 +126,11 @@ response.body
 
 ## Changelog
 
+* *v 0.2.9*
+    * Added agent attribute for redirect follow location
+    * Used 302 http answer code and header location for redirecting
+    * Used meta refresh url
+    * Refactored code for rubocop
 * *v 0.2.8*
     * Added processing Accept headers
 * *v 0.2.7*

data/Rakefile

@@ -1,6 +1,6 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: :spec

data/lib/tiny_grabber/agent.rb

@@ -2,334 +2,340 @@
 # Initialize connect with Resource
 # Setting connect attributes
 #
-class TinyGrabber::Agent
-  # Debug configuration
-  attr_accessor :debug
-  # Max time to execute request
-  attr_accessor :read_timeout
-  # Web browser name
-  attr_accessor :user_agent
-  # Remote proxy configuration
-  attr_accessor :proxy
-  # Basic authentification configuration
-  attr_accessor :basic_auth
-  # Headers
-  attr_accessor :headers
-  # Headers
-  attr_accessor :cookies
-
-  #  Agent aliases given from http://www.useragentstring.com/pages/Chrome/
-  AGENT_ALIASES = [
-    # Chrome
-    'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36',
-    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36',
-    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36',
-    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36',
-    # Firefox
-    'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1',
-    'Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0',
-    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:33.0) Gecko/20100101 Firefox/33.0',
-    'Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/31.0',
-    # Internet Explorer
-    'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko',
-    'Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0',
-    'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 7.0; InfoPath.3; .NET CLR 3.1.40767; Trident/6.0; en-IN)',
-    'Mozilla/5.0 (compatible; MSIE 10.0; Macintosh; Intel Mac OS X 10_7_3; Trident/6.0)',
-    # Opera
-    'Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16',
-    'Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14',
-    'Mozilla/5.0 (Windows NT 6.0; rv:2.0) Gecko/20100101 Firefox/4.0 Opera 12.14',
-    'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52',
-  ]
-
-  # Initialization object
-  #
-  def initialize
-    @debug = Debug.new
-
-    # Initialize variables agent attributes
-    @user_agent = AGENT_ALIASES[rand(AGENT_ALIASES.count) - 1]
-    @proxy = []
-    @basic_auth = {}
-    @headers = {}
-    @cookies = nil
-    @read_timeout = 10
-    # Initialize variable for URI object
-    @uri = nil
-    # Initialize variable for Net::HTTP request object
-    @http = Net::HTTP
-    # Initialize variable for Net::HTTP response object
-    @response = nil
-    @verify_mode = OpenSSL::SSL::VERIFY_NONE
-  end
-
-
-  # Set debug configuration
-  #
-  # @param debug
-  #
-  def debug= debug
-    debug = var_to_sym(debug, true)
-    if debug.is_a?(Hash)
-      @debug.active = debug[:active]
-      @debug.destination = debug[:destination]
-      @debug.save_html = debug[:save_html]
-    elsif debug.is_a?(TrueClass)
-      @debug.active = true
+class TinyGrabber
+  class Agent
+    # Debug configuration
+    attr_writer :debug
+    # Max time to execute request
+    attr_writer :read_timeout
+    # Web browser name
+    attr_writer :user_agent
+    # Remote proxy configuration
+    attr_accessor :proxy
+    # Basic authentification configuration
+    attr_writer :basic_auth
+    # Headers
+    attr_writer :headers
+    # Headers
+    attr_writer :cookies
+    # Set verify mode
+    attr_writer :verify_mode
+    # Follow location
+    attr_writer :follow_location
+
+    #  Agent aliases given from http://www.useragentstring.com/pages/Chrome/
+    AGENT_ALIASES = [
+      # Chrome
+      'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36',
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36',
+      'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36',
+      'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36',
+      # Firefox
+      'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1',
+      'Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0',
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:33.0) Gecko/20100101 Firefox/33.0',
+      'Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/31.0',
+      # Internet Explorer
+      'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko',
+      'Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0',
+      'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 7.0; InfoPath.3; .NET CLR 3.1.40767; Trident/6.0; en-IN)',
+      'Mozilla/5.0 (compatible; MSIE 10.0; Macintosh; Intel Mac OS X 10_7_3; Trident/6.0)',
+      # Opera
+      'Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16',
+      'Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14',
+      'Mozilla/5.0 (Windows NT 6.0; rv:2.0) Gecko/20100101 Firefox/4.0 Opera 12.14',
+      'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52'
+    ].freeze
+
+    # Initialization object
+    #
+    def initialize
+      @debug = Debug.new
+
+      # Initialize variables agent attributes
+      @user_agent = AGENT_ALIASES[rand(AGENT_ALIASES.count) - 1]
+      @proxy = []
+      @basic_auth = {}
+      @headers = {}
+      @cookies = nil
+      @follow_location = false
+      @read_timeout = 10
+      # Initialize variable for URI object
+      @uri = nil
+      # Initialize variable for Net::HTTP request object
+      @http = Net::HTTP
+      # Initialize variable for Net::HTTP response object
+      @response = nil
+      @verify_mode = OpenSSL::SSL::VERIFY_NONE
     end
-  end
-
 
-  # Set READ_TIMEOUT agent attribute
-  #
-  # @param read_timeout Waiting time to reading
-  #
-  def read_timeout= read_timeout
-    fail 'attribute read_timeout must be Integer' unless read_timeout.is_a?(Integer)
-    @read_timeout = read_timeout
-  end
-
-
-  # Set USER_AGENT agent attribute
-  #
-  # @param user_agent Web browser name
-  #
-  def user_agent= user_agent
-    fail 'attribute user_agent must be String' unless user_agent.is_a?(String)
-    @user_agent = user_agent
-  end
-
-
-  # Initialize Net::HTTP connection through proxy provider
-  # TYPE attribute distribute proxy type on SOCKS4(5) and HTTP(s)
-  #
-  # @param proxy Proxy configuration
-  #
-  def proxy= proxy
-    if proxy.is_a?(String)
-      ip, port, type = proxy.split(':')
-      fail 'attribute proxy must be in format ip:port' unless ip and port
-      type ||= :http
-      proxy = { ip: ip, port: port, type: type }
+    # Set debug configuration
+    #
+    # @param debug
+    #
+    def debug=(debug)
+      debug = var_to_sym(debug, true)
+      if debug.is_a?(Hash)
+        @debug.active = debug[:active]
+        @debug.destination = debug[:destination]
+        @debug.save_html = debug[:save_html]
+      elsif debug.is_a?(TrueClass)
+        @debug.active = true
+      end
     end
-    proxy = var_to_sym(proxy)
-    fail 'attribute proxy must be Hash' unless proxy.is_a?(Hash)
-    fail 'attribute proxy must contain :ip and :port keys' unless proxy[:ip] and proxy[:port]
 
-    @proxy = proxy
-    if [:socks, 'socks'].include? proxy[:type]
-      @http = Net::HTTP.SOCKSProxy(proxy[:ip].to_s, proxy[:port].to_s)
-    else
-      @http = Net::HTTP::Proxy(proxy[:ip], proxy[:port])
+    # Set READ_TIMEOUT agent attribute
+    #
+    # @param read_timeout Waiting time to reading
+    #
+    def read_timeout=(read_timeout)
+      raise 'attribute read_timeout must be Integer' unless read_timeout.is_a?(Integer)
+      @read_timeout = read_timeout
     end
-  end
-
-
-  # Set BASIC_AUTH agent attribute
-  #
-  # @param basic_auth Authentification configuration
-  #
-  def basic_auth= basic_auth
-    basic_auth = var_to_sym(basic_auth)
-    fail 'attribute basic_auth must be Hash' unless basic_auth.is_a?(Hash)
-    fail 'attribute basic_auth must contain :username and :password keys' unless basic_auth[:username] and basic_auth[:password]
-    @basic_auth = basic_auth
-  end
-
-
-  # Set HEADERS agent attribute
-  #
-  # @param headers Request headers
-  #
-  def headers= headers
-    fail 'attribute headers must be Hash' unless headers.is_a?(Hash)
-    @headers = headers
-  end
 
+    # Set USER_AGENT agent attribute
+    #
+    # @param user_agent Web browser name
+    #
+    def user_agent=(user_agent)
+      raise 'attribute user_agent must be String' unless user_agent.is_a?(String)
+      @user_agent = user_agent
+    end
 
-  # Set COOKIES agent attribute
-  #
-  # @param cookies Request cookies
-  #
-  def cookies= cookies
-    cookies = var_to_sym(cookies)
-    cookies = cookies.to_a.map { |x| "#{x[0]}=#{x[1]}" }.join('&') if cookies.is_a?(Hash)
-    fail 'attribute cookies must be String' unless cookies.is_a?(String)
-    @cookies = cookies
-  end
+    # Initialize Net::HTTP connection through proxy provider
+    # TYPE attribute distribute proxy type on SOCKS4(5) and HTTP(s)
+    #
+    # @param proxy Proxy configuration
+    #
+    def proxy=(proxy)
+      if proxy.is_a?(String)
+        ip, port, type = proxy.split(':')
+        raise 'attribute proxy must be in format ip:port' unless ip && port
+        type ||= :http
+        proxy = { ip: ip, port: port, type: type }
+      end
+      proxy = var_to_sym(proxy)
+      raise 'attribute proxy must be Hash' unless proxy.is_a?(Hash)
+      raise 'attribute proxy must contain :ip and :port keys' unless proxy[:ip] && proxy[:port]
+
+      @proxy = proxy
+      @http = if [:socks, 'socks'].include? proxy[:type]
+                Net::HTTP.SOCKSProxy(proxy[:ip].to_s, proxy[:port].to_s)
+              else
+                Net::HTTP::Proxy(proxy[:ip], proxy[:port])
+              end
+    end
 
+    # Set BASIC_AUTH agent attribute
+    #
+    # @param basic_auth Authentification configuration
+    #
+    def basic_auth=(basic_auth)
+      basic_auth = var_to_sym(basic_auth)
+      raise 'attribute basic_auth must be Hash' unless basic_auth.is_a?(Hash)
+      raise 'attribute basic_auth must contain :username and :password keys' unless basic_auth[:username] && basic_auth[:password]
+      @basic_auth = basic_auth
+    end
 
-  # Set verify_mode
-  #
-  # @param verify_mode SSL verify_mode
-  #
-  def verify_mode= verify_mode
-    @verify_mode = verify_mode
-  end
+    # Set HEADERS agent attribute
+    #
+    # @param headers Request headers
+    #
+    def headers=(headers)
+      raise 'attribute headers must be Hash' unless headers.is_a?(Hash)
+      @headers = headers
+    end
 
+    # Set COOKIES agent attribute
+    #
+    # @param cookies Request cookies
+    #
+    def cookies=(cookies)
+      cookies = var_to_sym(cookies)
+      cookies = cookies.to_a.map { |x| "#{x[0]}=#{x[1]}" }.join('&') if cookies.is_a?(Hash)
+      raise 'attribute cookies must be String' unless cookies.is_a?(String)
+      @cookies = cookies
+    end
 
-  # Fetch request for GET and POST HTTP methods
-  # Setting USER_AGENT, BASIC_AUTH, HEADERS, COOKIES request attribute
-  # Make response and save COOKIES for next requests
-  #
-  # @param url Resource link
-  # @param method Request method
-  # @param headers Request header
-  # @param params Request additional params
-  #
-  def fetch url, method = :get, headers = {}, params = {}
-    if @debug.active
-      @debug.save '=============================='
-      @debug.save "#{method.upcase} #{url}"
-      @debug.save "-> [proxy] = #{@proxy}" if @proxy
-      @debug.save "-> [params] = #{params}"
-      @debug.save '------------------------------'
+    # Set verify_mode
+    #
+    # @param verify_mode SSL verify_mode
+    #
+    # def verify_mode=(verify_mode)
+    #   @verify_mode = verify_mode
+    # end
+
+    # Init follow location for redirect
+    #
+    # @param follow_location Follow location flag
+    #
+    def follow_location=(follow_location)
+      raise 'attribute follow_location must be Boolean' unless follow_location.is_a?(TrueClass) || follow_location.is_a?(FalseClass)
+      @follow_location = follow_location
     end
-    set_uri url
-    case method
+
+    # Fetch request for GET and POST HTTP methods
+    # Setting USER_AGENT, BASIC_AUTH, HEADERS, COOKIES request attribute
+    # Make response and save COOKIES for next requests
+    #
+    # @param url Resource link
+    # @param method Request method
+    # @param headers Request header
+    # @param params Request additional params
+    #
+    def fetch(url, method = :get, headers = {}, params = {})
+      if @debug.active
+        @debug.save '=============================='
+        @debug.save "#{method.upcase} #{url}"
+        @debug.save "-> [proxy] = #{@proxy}" if @proxy
+        @debug.save "-> [params] = #{params}"
+        @debug.save '------------------------------'
+      end
+      convert_to_uri url
+      case method
       when :get
         @request = Net::HTTP::Get.new(@uri.request_uri)
       when :post
         @request = Net::HTTP::Post.new(@uri.request_uri)
         @request.set_form_data(params)
-    end
-    set_user_agent if @user_agent
-    set_basic_auth unless @basic_auth.empty?
-    @headers = headers unless headers.empty?
-    set_headers unless @headers.empty?
-    set_cookies if @cookies
-    @response = send_request
-    case @response
+      end
+      set_user_agent if @user_agent
+      set_basic_auth unless @basic_auth.empty?
+      @headers = headers unless headers.empty?
+      set_headers unless @headers.empty?
+      set_cookies if @cookies
+      @response = send_request
+      case @response
       # HTTP response code 1xx
       when Net::HTTPInformation
-        @debug.save "<- [response] = Net::HTTPInformation" if @debug.active
+        @debug.save '<- [response] = Net::HTTPInformation' if @debug.active
       # HTTP response code 2xx
       when Net::HTTPSuccess
         save_headers if @response.header
         save_cookies if @response.cookies
         @debug.save "<- [response] = #{@response.code} Net::HTTPSuccess" if @debug.active
+        # Follow meta refresh
+        if @follow_location
+          refresh = @response.ng.at_css('meta[http-equiv="refresh"]')
+          @response = fetch refresh.attr('content').gsub(/\A.*?(http)/, 'http') if refresh
+        end
       # HTTP response code 3xx
       when Net::HTTPRedirection
         @debug.save "<- [response] = #{@response.code} Net::HTTPRedirection" if @debug.active
+        @debug.save 'try curl user_agent: tg.user_agent=\'curl\''
+        # Follow location
+        @response = fetch @response.header['Location'] if @follow_location
       # HTTP response code 4xx
       when Net::HTTPClientError
         @debug.save "<- [response] = #{@response.code} Net::HTTPClientError" if @debug.active
       # HTTP response code 5xx
       when Net::HTTPServerError
         @debug.save "<- [response] = #{@response.code} Net::HTTPServerError" if @debug.active
+      end
+      @debug.save_to_file @response.body if @debug.save_html
+      @response
     end
-    @debug.save_to_file @response.body  if @debug.save_html
-    @response
-  end
-
-
-  # Initialize URI object from request url
-  #
-  # @param url Request link
-  #
-  def set_uri url
-    # It's magic work with escaped url
-    @uri = URI(URI.escape(URI.unescape(url)))
-    @debug.save "-> [uri] = #{@uri}" if @debug.active
-  end
-
-
-  # Set USER_AGENT request attribute
-  #
-  def set_user_agent
-    @headers['User-Agent'] = @user_agent
-    @debug.save "-> [user_agent] = #{@user_agent}" if @debug.active
-  end
 
+    # Initialize URI object from request url
+    #
+    # @param url Request link
+    #
+    def convert_to_uri(url)
+      # It's magic work with escaped url
+      @uri = URI(URI.escape(URI.unescape(url)))
+      @debug.save "-> [uri] = #{@uri}" if @debug.active
+    end
 
-  # Set BASIC_AUTH request authentification
-  #
-  def set_basic_auth
-    @request.basic_auth @basic_auth[:username], @basic_auth[:password]
-    @debug.save "-> [basic_auth] = #{@basic_auth}" if @debug.active
-  end
+    # Set USER_AGENT request attribute
+    #
+    def set_user_agent
+      @headers['User-Agent'] = @user_agent
+      @debug.save "-> [user_agent] = #{@user_agent}" if @debug.active
+    end
 
+    # Set BASIC_AUTH request authentification
+    #
+    def set_basic_auth
+      @request.basic_auth @basic_auth[:username], @basic_auth[:password]
+      @debug.save "-> [basic_auth] = #{@basic_auth}" if @debug.active
+    end
 
-  # Set request HEADERS
-  #
-  def set_headers
-    @headers.each do |k, v|
-      k = String(k)
-      case k
+    # Set request HEADERS
+    #
+    def set_headers
+      @headers.each do |k, v|
+        k = String(k)
+        case k
         when 'Accept'
           @request[k] = v
         else
           @request.add_field(k, v)
+        end
       end
+      @debug.save "-> [headers] = #{@headers}" if @debug.active
     end
-    @debug.save "-> [headers] = #{@headers}" if @debug.active
-  end
-
 
-  # Set request COOKIES
-  #
-  def set_cookies
-    @request['Cookie'] = @cookies
-    @debug.save "-> [cookies] = #{@cookies}" if @debug.active
-  end
-
-
-  # Send request and get response
-  # Use SSL connect for HTTPS link scheme
-  #
-  def send_request
-    @http.start(@uri.host, @uri.port, use_ssl: @uri.scheme == 'https', verify_mode: @verify_mode, read_timeout: @read_timeout) do |http|
-      @debug.save "-> [read_timeout] = #{@read_timeout}" if @debug.active
-      http.request(@request)
+    # Set request COOKIES
+    #
+    def set_cookies
+      @request['Cookie'] = @cookies
+      @debug.save "-> [cookies] = #{@cookies}" if @debug.active
     end
-  end
-
-
-  # Save response headers in agent attribute
-  #
-  def save_headers
-    @headers = @response.headers
-    # Delete header TRANSFER_ENCODING for chain of requests
-    @headers.delete('transfer-encoding')
-    @debug.save "<- [headers] = #{@headers}" if @debug.active
-  end
 
+    # Send request and get response
+    # Use SSL connect for HTTPS link scheme
+    #
+    def send_request
+      @http.start(@uri.host, @uri.port, use_ssl: @uri.scheme == 'https', verify_mode: @verify_mode, read_timeout: @read_timeout) do |http|
+        @debug.save "-> [read_timeout] = #{@read_timeout}" if @debug.active
+        http.request(@request)
+      end
+    end
 
-  # Save response cookies in agent attribute
-  #
-  def save_cookies
-    @cookies = @response.cookies
-    @debug.save "<- [cookies] = #{@cookies}" if @debug.active
-  end
+    # Save response headers in agent attribute
+    #
+    def save_headers
+      @headers = @response.headers
+      # Delete header TRANSFER_ENCODING for chain of requests
+      @headers.delete('transfer-encoding')
+      @debug.save "<- [headers] = #{@headers}" if @debug.active
+    end
 
+    # Save response cookies in agent attribute
+    #
+    def save_cookies
+      @cookies = @response.cookies
+      @debug.save "<- [cookies] = #{@cookies}" if @debug.active
+    end
 
-  # Clears headers and cookies
-  #
-  def reset
-    @headers = {}
-    @cookies = nil
-  end
+    # Clears headers and cookies
+    #
+    def reset
+      @headers = {}
+      @cookies = nil
+    end
 
-  # Convert variables and contains to symbol
-  #
-  # @param var Variable need to convert
-  #
-  def var_to_sym var, str_to_sym = false
-    if var.is_a?(Hash)
-      result = {}
-      var.each do |k, v|
-        result[k.to_sym] = var_to_sym(v, str_to_sym)
-      end
-    elsif var.is_a?(Array)
-      result = []
-      var.each do |v|
-        result << var_to_sym(v, str_to_sym)
+    # Convert variables and contains to symbol
+    #
+    # @param var Variable need to convert
+    #
+    def var_to_sym(var, str_to_sym = false)
+      if var.is_a?(Hash)
+        result = {}
+        var.each do |k, v|
+          result[k.to_sym] = var_to_sym(v, str_to_sym)
+        end
+      elsif var.is_a?(Array)
+        result = []
+        var.each do |v|
+          result << var_to_sym(v, str_to_sym)
+        end
+      elsif var.is_a?(String)
+        result = str_to_sym ? var.to_sym : var
+      else
+        result = var
       end
-    elsif var.is_a?(String)
-      result = str_to_sym ? var.to_sym : var
-    else
-      result = var
+      result
     end
-    result
   end
-end
+end

data/lib/tiny_grabber/debug.rb

@@ -15,76 +15,30 @@ class Debug
     @save_html = false
   end
 
-
-  # Set debug active flag
-  #
-  # @param active Flag
-  #
-  def active= active
-    @active = active
-  end
-
-
-  # Get debug active flag
-  def active
-    @active
-  end
-
-
-  # Set debug destination
-  #
-  # @param destination Save log to file or print
-  #
-  def destination= destination
-    @destination = destination
-  end
-
-
-  # Get debug destination
-  #
-  def destination
-    @destination
-  end
-
-
-  # Set debug flag to save response HTML to file
-  #
-  # @param save_html Flag
-  def save_html= save_html
-    @save_html = save_html
-  end
-
-
-  # Get debug flag to save response HTML to file
-  #
-  def save_html
-    @save_html
-  end
-
-
   # Save log information
   #
   # @param message Message body
   #
-  def save message
+  def save(message)
     message = "TG | #{Time.now.strftime('%Y%m%d-%H%M%S')} | #{message}"
     case @destination
-      when :file
-        save_to_file message
-      when :print
-        p message
+    when :file
+      save_to_file message
+    when :print
+      p message
     end
   end
 
-
   # Save log information to file
   #
   # @param message Message body
   #
-  def save_to_file message
+  def save_to_file(message)
+    # Encode message for correct Unix encoding
+    message = message.force_encoding('utf-8')
     debug_path = "#{Dir.pwd}/log"
-    Dir.mkdir(debug_path, 0775) unless File.exists? debug_path
+    Dir.mkdir(debug_path, 0o775) unless File.exist? debug_path
     filename = "#{Time.now.strftime('%Y%m%d')}.log"
     File.open("#{debug_path}/#{filename}", 'a+') { |f| f << "#{message}\r\n" }
   end
-end
+end

data/lib/tiny_grabber/http.rb

@@ -4,29 +4,27 @@ require 'nokogiri'
 module Net
   # Success response class
   class HTTPOK
-
     # Nokogiri object of response
     #
     def ng
-      Nokogiri::HTML(self.body)
+      Nokogiri::HTML(body)
     end
 
     # Response Cookies
     #
     def cookies
-      cookies = self.get_fields('set-cookie')
+      cookies = get_fields('set-cookie')
       if cookies
         cookies.map { |cookie| cookie.gsub(/\A([^;]+).*\Z/, '\1') }.join('&')
-      else
-        nil
       end
     end
 
-
     # Response Headers
     #
     def headers
-      self.header.to_hash.inject({}) { |headers, (header_key, header_value)| headers[header_key] = header_value.first; headers }
+      header.to_hash.each_with_object({}) do |header_key, header_value|
+        header_value[header_key] = header_value.first
+      end
     end
   end
-end
+end

data/lib/tiny_grabber/version.rb

@@ -1,4 +1,4 @@
 class TinyGrabber
   # Version number
-  VERSION = '0.2.8'
+  VERSION = '0.3.0'.freeze
 end

data/lib/tiny_grabber.rb

@@ -10,17 +10,15 @@ require 'tiny_grabber/http'
 # Main class for TinyGrabber
 #
 class TinyGrabber
-
   # Initialize a new TinyGrabber user agent.
   #
   def initialize
     @agent = TinyGrabber::Agent.new
   end
 
-
   # Singleton > Initialize a new TinyGrabber user agent.
   #
-  def self.initialize config = {}
+  def self.initialize(config = {})
     @agent = TinyGrabber::Agent.new
 
     @agent.debug = config[:debug] if config[:debug]
@@ -32,13 +30,12 @@ class TinyGrabber
     @agent.cookies = config[:cookies] if config[:cookies]
   end
 
-
   # HTTP::GET request
   #
   # @param url Resource link
   # @param headers Request header
   #
-  def get url, headers = {}
+  def get(url, headers = {})
     @agent.fetch url, :get, headers
   end
 
@@ -47,129 +44,115 @@ class TinyGrabber
   # @param url Resource link
   # @param headers Request header
   #
-  def self.get url, config = {}
-    initialize  config
+  def self.get(url, config = {})
+    initialize config
     @agent.fetch url, :get
   end
 
-
   # HTTP::POST request
   #
   # @param url Resource link
   # @param params Request post data
   # @param headers Request header
   #
-  def post url, params = {}, headers = {}
+  def post(url, params = {}, headers = {})
     @agent.fetch url, :post, headers, params
   end
 
-
   # Singleton >  HTTP::GET request
   #
   # @param url Resource link
   # @param headers Request header
   #
-  def self.post url, params = {}, config = {}
-    initialize  config
+  def self.post(url, params = {}, config = {})
+    initialize config
     @agent.fetch url, :post, {}, params
   end
 
-
   # Set DEBUG flag
   #
   # @param debug Flag to start debug
   #
-  def debug= debug
+  def debug=(debug)
     @agent.debug = debug
   end
 
-
   # Read READ_TIMEOUT agent attribute
   #
   def read_timeout
     @agent.read_timeout
   end
 
-
   # Set READ_TIMEOUT agent attribute
   #
   # @param read_timeout Waiting time to reading
   #
-  def read_timeout= read_timeout
+  def read_timeout=(read_timeout)
     @agent.read_timeout = read_timeout
   end
 
-
   # Read USER_AGENT agent attribute
   #
   def user_agent
     @agent.user_agent
   end
 
-
   # Set USER_AGENT agent attribute
   #
   # @param user_agent Web browser name
   #
-  def user_agent= user_agent
+  def user_agent=(user_agent)
     @agent.user_agent = user_agent
   end
 
-
   # Read PROXY agent attribute
   #
   def proxy
     @agent.proxy
   end
 
-
   # Set PROXY agent attribute
   #
   # @param proxy Proxy configuration
   #
-  def proxy= proxy
+  def proxy=(proxy)
     @agent.proxy = proxy
   end
 
-
   # Set BASIC_AUTH agent attribute
   #
   # @param username Authentification username
   # @param password Authentification password
   #
-  def basic_auth username, password
+  def basic_auth(username, password)
     @agent.basic_auth = { username: username, password: password }
   end
 
-
   # Read HEADERS agent attribute
   #
   def headers
     @agent.headers
   end
 
-
   # Set HEADERS agent attribute
   #
   # @param headers Request headers
   #
-  def headers= headers
+  def headers=(headers)
     @agent.headers = headers
   end
 
-
   # Read COOKIES agent attribute
   #
   def cookies
     @agent.cookies
   end
 
-
   # Set COOKIES agent attribute
   #
   # @param cookies Request cookies
   #
-  def cookies= cookies
+  def cookies=(cookies)
     @agent.cookies = cookies
   end
 
@@ -179,12 +162,19 @@ class TinyGrabber
     @agent.reset
   end
 
-
   # Set verify_mode
   #
   # @param verify_mode SSL verify mode
   #
-  def verify_mode= verify_mode
+  def verify_mode=(verify_mode)
     @agent.verify_mode = verify_mode
   end
-end
+
+  # Set follow_location
+  #
+  # @param follow_location Follow location flag
+  #
+  def follow_location=(follow_location)
+    @agent.follow_location = follow_location
+  end
+end

data/tiny_grabber.gemspec

@@ -6,11 +6,11 @@ require 'tiny_grabber/version'
 Gem::Specification.new do |spec|
   spec.name          = 'tiny_grabber'
   spec.version       = TinyGrabber::VERSION
-  spec.authors       = ["Aleksandr Chernyshev"]
+  spec.authors       = ['Aleksandr Chernyshev']
   spec.email         = ['moroznoeytpo@gmail.com']
 
-  spec.summary       = %q{Tiny grabber}
-  spec.description   = %q{Simple gem for grabbing remote web page.}
+  spec.summary       = 'Tiny grabber'
+  spec.description   = 'Simple gem for grabbing remote web page.'
   spec.homepage      = 'https://github.com/moroznoeytpo/tiny_grabber'
   spec.license       = 'MIT'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tiny_grabber
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.3.0
 platform: ruby
 authors:
 - Aleksandr Chernyshev
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-04 00:00:00.000000000 Z
+date: 2016-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: socksify