tiny_grabber 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9125f8132b04b1a08e64a47abf7ccb5085688b55
-  data.tar.gz: 050aad4742cdeeba67cfc64dad8c6d9c0a02ec97
+  metadata.gz: 99211b6e18440ea58661cf5b5a5bdadec631f942
+  data.tar.gz: 1e78bd2089d65949431b519ae7a15b3f4bba202f
 SHA512:
-  metadata.gz: d1ff3deb449bc1aedf86b2886156043decf28cf327ec25e30cd4725f07f5dc82faa01c803aa161bf573e3799f747a15673919fb470db75546810a932ee513093
-  data.tar.gz: c977e50867454e3af09e00e6b9f69f5b8efc7b22aaa0127b3c54c8aee206692359322a0c4418b81ab23bbc4a54a562f197f9fc98088225d97a70d1d1699d83f2
+  metadata.gz: 3a4dc8a114ea9818dee2bd400ccc5e45c60ae40f2a9e7ce5002c51b214b9e471d1066f2d865e48ab0acbe91e80c51d55cb17ed43a7eed29e01283f82c61c09b8
+  data.tar.gz: 4ec0f357261f447f3cd34dda16a9376dd770a73a9c7fc30b22577f42de1b4a33bd557b5c8da85ce54a5d40d780af4e152ebbd93b0d50097fbf0c7081d134f30e

data/.gitignore

@@ -5,6 +5,6 @@
 /coverage/
 /doc/
 /pkg/
-/spec/reports/
+/spec/tiny_grabber_spec.rb
 /tmp/
 /log/

data/README.md

@@ -27,44 +27,62 @@ Or install it yourself as:
 
 require 'tiny_grabber'
 
-# Link to remote site
-url = 'https://github.com/moroznoeytpo/tiny_grabber'
+read_timeout = 300
+user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.94 Safari/537.36'
+proxy = { ip: 'xx.xx.xx.xx', port: 'xxxx' }
+headers1 = { 'Content-Type' => 'text/html; charset=utf-8' }
+headers2 = { 'Content-Type' => 'text/html; charset=utf-8', 'Connection' => 'keep-alive' }
+cookies = 'username=username&password=password'
+params = { key: 'value' }
+
+# Initialize TinyGrabber object
+tg = TinyGrabber.new
+# Set debug flag for view log information
+tg.debug = true
+# Set max time to execute request
+tg.read_timeout = read_timeout
+# Set web browser name
+tg.user_agent = user_agent
+# Set proxy configuration
+tg.proxy = proxy
+# Set basic authentification
+tg.basic_auth('username', 'password')
+# Set HTTP headers
+tg.headers = headers1
+# Set HTTP cookies
+tg.cookies = cookies
+
+# Make response with GET method
+response = tg.get 'https://whoer.net/ru', headers
+# Make response with POST method
+response = tg.get 'https://whoer.net/ru', params, headers
+
+# Make singleton response with GET method
+response = TinyGrabber.get 'https://whoer.net/ru', headers, { debug = true, read_timeout = read_timeout ...}
+# Make singleton response with POST method
+response = TinyGrabber.post 'https://whoer.net/ru', params, headers, { debug = true, read_timeout = read_timeout ...}
+
+# Get Nokogiri object from response HTML
+ng = response.ng
+# Get HTTP response code
+response.code
+# Get response cookies
+response.cookies
+# Get response headers
+response.headers
 
-# Set headers
-headers = { 'Content-Type' => 'application/json' }
-
-# Set http(s)/socks4(5) proxy
-# Proxy type by default is http. You can change it to socks, with setting params proxy_type equal socks
-proxy = { ip: 'xx.xx.xx.xx', port: xx, proxy_type: :socks }
-
-# Set Basic Authentication
-auth = { username: '', password: '' }
-
-# Set POST data
-# Request HTTP type by default is GET. You can send POST request with setting post params. Also you cat send empty POST request.
-post = { some_data: '' }
-
-# HTTP GET request
-response = TinyGrabber.get url, headers: headers, proxy: proxy, auth: auth
-
-# HTTP GET request
-response = TinyGrabber.post url, post, headers: headers, proxy: proxy, auth: auth
-
-# HTTP answer code
-p response.code
-
-# HTTP content
-p response.body
-
-# Nokogiri object
-p response.ng
-
-# Response cookies
-p response.cookies
 ```
 
 ## Changelog
 
+* *v 0.2.0*
+    * Now there is an opportunity to create object TinyGrabber
+    * Change order of parameters for singleton request
+    * Add response cookies and headers
+    * Add debug flag for detilazition log and save result HTML to /log/*.html file
+
+* *v 0.1.1*
+    * Save cookie in Redis
 * *v 0.1.0*
     * Add TinyGrabber.post method for HTTP POST request
 * *v 0.0.7*

data/lib/tiny_grabber/agent.rb

@@ -0,0 +1,245 @@
+# Net::HTTP agent for TinyGrabber
+# Initialize connect with Resource
+# Setting connect attributes
+#
+class TinyGrabber::Agent
+  # Debug flag for detilazition log and save result HTML to /log/*.html file
+  attr_accessor :debug
+  # Max time to execute request
+  attr_accessor :read_timeout
+  # Web browser name
+  attr_accessor :user_agent
+  # Remote proxy configuration
+  attr_accessor :proxy
+  # Basic authentification configuration
+  attr_accessor :basic_auth
+  # Headers
+  attr_accessor :headers
+  # Headers
+  attr_accessor :cookies
+
+  # Initialization object
+  #
+  def initialize
+    @debug = false
+
+    # Initialize variables agent attributes
+    @user_agent = nil
+    @proxy = []
+    @basic_auth = {}
+    @headers = {}
+    @cookies = nil
+    @read_timeout = 10
+    # Initialize variable for URI object
+    @uri = nil
+    # Initialize variable for Net::HTTP request object
+    @http = Net::HTTP
+    # Initialize variable for Net::HTTP response object
+    @response = nil
+  end
+
+
+  # Set READ_TIMEOUT agent attribute
+  #
+  # @param read_timeout Waiting time to reading
+  #
+  def read_timeout= read_timeout
+    fail 'attribute read_timeout must be Integer' unless read_timeout.is_a?(Integer)
+    @read_timeout = read_timeout
+  end
+
+
+  # Set USER_AGENT agent attribute
+  #
+  # @param user_agent Web browser name
+  #
+  def user_agent= user_agent
+    fail 'attribute user_agent must be String' unless user_agent.is_a?(String)
+    @user_agent = user_agent
+  end
+
+
+  # Initialize Net::HTTP connection through proxy provider
+  # TYPE attribute distribute proxy type on SOCKS4(5) and HTTP(s)
+  #
+  # @param proxy Proxy configuration
+  #
+  def proxy= proxy
+    if proxy.is_a?(String)
+      ip, port = proxy.split(':')
+      fail 'attribute proxy must be in format ip:port' unless ip and port
+      proxy = { ip: ip, port: port }
+    end
+    fail 'attribute proxy must be Hash' unless proxy.is_a?(Hash)
+    fail 'attribute proxy must contain :ip and :port keys' unless proxy[:ip] and proxy[:port]
+
+    @proxy = proxy
+    if ['socks', :socks].include?(proxy[:type])
+      @http = Net::HTTP.SOCKSProxy(proxy[:ip], proxy[:port])
+    else
+      @http = Net::HTTP::Proxy(proxy[:ip], proxy[:port])
+    end
+  end
+
+
+  # Set BASIC_AUTH agent attribute
+  #
+  # @param basic_auth Authentification configuration
+  #
+  def basic_auth= basic_auth
+    fail 'attribute basic_auth must be Hash' unless basic_auth.is_a?(Hash)
+    fail 'attribute basic_auth must contain :username and :password keys' unless basic_auth[:username] and basic_auth[:password]
+    @basic_auth = basic_auth
+  end
+
+
+  # Set HEADERS agent attribute
+  #
+  # @param headers Request headers
+  #
+  def headers= headers
+    fail 'attribute headers must be Hash' unless headers.is_a?(Hash)
+    @headers = headers
+  end
+
+
+  # Set COOKIES agent attribute
+  #
+  # @param cookies Request cookies
+  #
+  def cookies= cookies
+    fail 'attribute cookies must be String' unless cookies.is_a?(String)
+    @cookies = cookies
+  end
+
+
+  # Fetch request for GET and POST HTTP methods
+  # Setting USER_AGENT, BASIC_AUTH, HEADERS, COOKIES request attribute
+  # Make response and save COOKIES for next requests
+  #
+  # @param url Resource link
+  # @param method Request method
+  # @param headers Request header
+  # @param params Request additional params
+  #
+  def fetch url, method = :get, headers = {}, params = {}
+    if @debug
+      p "#{debug_initial_word} =============================="
+      p "#{debug_initial_word} #{method.upcase} #{url}"
+      p "#{debug_initial_word} #{params}"
+      p "#{debug_initial_word} ------------------------------"
+    end
+    set_uri url
+    case method
+      when :get
+        @request = Net::HTTP::Get.new(@uri.request_uri)
+      when :post
+        @request = Net::HTTP::Post.new(@uri.request_uri)
+        @request.set_form_data(params)
+    end
+    set_user_agent if @user_agent
+    set_basic_auth unless @basic_auth.empty?
+    @headers = headers unless headers.empty?
+    set_headers unless @headers.empty?
+    set_cookies if @cookies
+    @response = send_request
+    case @response
+      # HTTP response code 1xx
+      when Net::HTTPInformation
+      # HTTP response code 2xx
+      when Net::HTTPSuccess
+        save_headers if @response.header
+        save_cookies if @response.cookies
+      # HTTP response code 3xx
+      when Net::HTTPRedirection
+      # HTTP response code 4xx
+      when Net::HTTPClientError
+      # HTTP response code 5xx
+      when Net::HTTPServerError
+    end
+    if @debug
+      debug_filename = "log/#{method.upcase}_#{@uri.to_s.gsub(/[\/:]/, '_').gsub(/_+/, '_')}"
+      File.open(debug_filename, 'wb') { |f| f << @response.body } if @debug
+      p "#{debug_initial_word} HTML > #{debug_filename}"
+    end
+    @response
+  end
+
+
+  # Initialize URI object from request url
+  #
+  # @param url Request link
+  #
+  def set_uri url
+    # It's magic work with escaped url
+    @uri = URI(URI.escape(URI.unescape(url)))
+    p "#{debug_initial_word} URI = #{@uri}" if @debug
+  end
+
+
+  # Set USER_AGENT request attribute
+  #
+  def set_user_agent
+    @headers['User-Agent'] = @user_agent
+    p "#{debug_initial_word} user_agent = #{@user_agent}" if @debug
+  end
+
+
+  # Set BASIC_AUTH request authentification
+  #
+  def set_basic_auth
+    @request.basic_auth @basic_auth[:username], @basic_auth[:password]
+    p "#{debug_initial_word} basic_auth = #{@basic_auth}" if @debug
+  end
+
+
+  # Set request HEADERS
+  #
+  def set_headers
+    @headers.each { |k, v| @request.add_field(String(k), v) }
+    p "#{debug_initial_word} headers = #{@headers}" if @debug
+  end
+
+
+  # Set request COOKIES
+  #
+  def set_cookies
+    @request['Cookie'] = @cookies
+    p "#{debug_initial_word} cookies = #{@cookies}" if @debug
+  end
+
+
+  # Send request and get response
+  # Use SSL connect for HTTPS link scheme
+  #
+  def send_request
+    @http.start(@uri.host, @uri.port, use_ssl: @uri.scheme == 'https') do |http|
+      http.read_timeout = @read_timeout
+      p "#{debug_initial_word} read_timeout = #{@read_timeout}" if @debug
+      http.request(@request)
+    end
+  end
+
+
+  # Save response headers in agent attribute
+  #
+  def save_headers
+    @headers = @response.headers
+    p "#{debug_initial_word} save_headers = #{@headers}" if @debug
+  end
+
+
+  # Save response cookies in agent attribute
+  #
+  def save_cookies
+    @cookies = @response.cookies
+    p "#{debug_initial_word} save_cookies = #{@cookies}" if @debug
+  end
+
+
+  # Tiny grabber initial word for debug
+  #
+  def debug_initial_word
+    "TG | #{Time.now.strftime('%Y%m%d-%H%M%S')} |"
+  end
+end

data/lib/tiny_grabber/http.rb

@@ -14,7 +14,19 @@ module Net
     # Response Cookies
     #
     def cookies
-      self.get_fields('set-cookie')
+      cookies = self.get_fields('set-cookie')
+      if cookies
+        cookies.map { |cookie| cookie.gsub(/\A([^;]+).*\Z/, '\1') }.join('&')
+      else
+        nil
+      end
+    end
+
+
+    # Response Headers
+    #
+    def headers
+      self.header.to_hash.inject({}) { |headers, (header_key, header_value)| headers[header_key] = header_value.first; headers }
     end
   end
 end

data/lib/tiny_grabber/version.rb

@@ -1,4 +1,4 @@
 class TinyGrabber
   # Version number
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/tiny_grabber.rb

@@ -1,147 +1,179 @@
-require "tiny_grabber/version"
+require 'tiny_grabber/version'
+require 'tiny_grabber/agent'
 
 require 'uri'
 require 'net/http'
 require 'socksify/http'
 require 'tiny_grabber/http'
+require 'redis'
 
 # Main class for TinyGrabber
+#
 class TinyGrabber
 
-  @uri
-  @params
-  @http
-  @request
+  # Initialize a new TinyGrabber user agent.
+  #
+  def initialize
+    @agent = TinyGrabber::Agent.new
+  end
+
+
+  # Singleton > Initialize a new TinyGrabber user agent.
+  #
+  def self.initialize config = {}
+    @agent = TinyGrabber::Agent.new
+
+    @agent.debug = config[:debug] if config[:debug]
+    @agent.read_timeout = config[:read_timeout] if config[:read_timeout]
+    @agent.user_agent = config[:user_agent] if config[:user_agent]
+    @agent.proxy = config[:proxy] if config[:proxy]
+    @agent.basic_auth = config[:basic_auth] if config[:basic_auth]
+    @agent.headers = config[:headers] if config[:headers]
+    @agent.cookies = config[:cookies] if config[:cookies]
+  end
+
 
-  ##
-  # Get Net::HTTP object for content from remote single page
-  # [url (string)] Link to resource
-  # [params (hash)] Request params
-  # * [proxy (hash)] Configuration of remote proxy server
-  #   * *ip* address of remote proxy server
-  #   * *port* of remote proxy server
-  #   * *proxy_type* of remote proxy server
-  #     * *http* for http(s) proxy servers (by default)
-  #     * *socks* for socks4/5 proxy servers
-  # * [headers (hash)] Headers
-  # * [auth (hash)] Basic Authentication
-  #   * *username* Authenticate username
-  #   * *password* Authenticate password
+  # HTTP::GET request
+  #
+  # @param url Resource link
+  # @param headers Request header
   #
-  # = Example
-  #   TinyGrabber.get url, headers: { 'Content-Type' => 'application/json' }, proxy: { ip: 'xx.xx.xx.xx', port: xx, proxy_type: :socks }
+  def get url, headers = {}
+    @agent.fetch url, :get, headers
+  end
+
+  # Singleton > HTTP::GET request
   #
-  # @param url Link to resource
-  # @param params Request params
+  # @param url Resource link
+  # @param headers Request header
   #
-  def self.get url, params = {}
-    @uri = set_url url
-    @params = convert_params_to_sym params
-    @http = set_http_connect
-    @request = Net::HTTP::Get.new(@uri.request_uri)
-    set_basic_auth
-    set_headers
-    get_request
+  def self.get url, headers = {}, config = {}
+    initialize  config
+    @agent.fetch url, :get, headers
   end
 
 
-  ##
-  # Get Net::HTTP object for content from remote single page
-  # [url (string)] Link to resource
-  # [data (hash)] Post data
-  # [params (hash)] Request params
-  # * [proxy (hash)] Configuration of remote proxy server
-  #   * *ip* address of remote proxy server
-  #   * *port* of remote proxy server
-  #   * *proxy_type* of remote proxy server
-  #     * *http* for http(s) proxy servers (by default)
-  #     * *socks* for socks4/5 proxy servers
-  # * [headers (hash)] Headers
-  # * [auth (hash)] Basic Authentication
-  #   * *username* Authenticate username
-  #   * *password* Authenticate password
+  # HTTP::POST request
   #
-  # = Example
-  #   TinyGrabber.post url, data, headers: { 'Content-Type' => 'application/json' }, proxy: { ip: 'xx.xx.xx.xx', port: xx, proxy_type: :socks }
+  # @param url Resource link
+  # @param params Request post data
+  # @param headers Request header
   #
-  # @param url Link to resource
-  # @param params Request params
+  def post url, params = {}, headers = {}, config = {}
+    @agent.fetch url, :post, headers, params
+  end
+
+
+  # Singleton >  HTTP::GET request
+  #
+  # @param url Resource link
+  # @param headers Request header
   #
-  def self.post url, data, params = {}
-    @uri = set_url url
-    data = convert_params_to_sym data
-    @params = convert_params_to_sym params
-    @http = set_http_connect
-    @request = Net::HTTP::Post.new(@uri.request_uri)
-    @request.set_form_data(data)
-    set_basic_auth
-    set_headers
-    get_request
+  def self.post url, params = {}, headers = {}, config = {}
+    initialize  config
+    @agent.fetch url, :post, headers, params
   end
 
-  private
 
-  # Convert all params key to symbols
+  # Set DEBUG flag
   #
-  # @param params Request params
+  # @param debug Flag to start debug
   #
-  def self.convert_params_to_sym params
-    params.is_a?(Hash) ? params.inject({}) { |h, (k, v)| h[k.to_sym] = convert_params_to_sym v; h } : params
+  def debug= debug
+    @agent.debug = debug
   end
 
-  # Convert URL to URI
+
+  # Read READ_TIMEOUT agent attribute
   #
-  # @param url Link to resource
+  def read_timeout
+    @agent.read_timeout
+  end
+
+
+  # Set READ_TIMEOUT agent attribute
+  #
+  # @param read_timeout Waiting time to reading
   #
-  def self.set_url url
-    URI(URI.escape(url))
+  def read_timeout= read_timeout
+    @agent.read_timeout = read_timeout
   end
 
 
-  # Use proxy for request
+  # Read USER_AGENT agent attribute
   #
-  def self.set_http_connect
-    if @params[:proxy]
-      if ['socks', :socks].include?(@params[:proxy][:proxy_type])
-        Net::HTTP.SOCKSProxy(@params[:proxy][:ip], @params[:proxy][:port])
-      else
-        Net::HTTP::Proxy(@params[:proxy][:ip], @params[:proxy][:port])
-      end
-    else
-      Net::HTTP
-    end
+  def user_agent
+    @agent.user_agent
   end
 
 
-  # Set Basic Auth
+  # Set USER_AGENT agent attribute
   #
-  # @param request Request object
-  # @param params Request params
+  # @param user_agent Web browser name
   #
-  def self.set_basic_auth
-    if @params[:auth]
-      @request.basic_auth @params[:auth][:username], @params[:auth][:password]
-    end
+  def user_agent= user_agent
+    @agent.user_agent = user_agent
   end
 
 
-  # Headers
+  # Read PROXY agent attribute
   #
-  # @param request Request object
-  # @param params Request params
+  def proxy
+    @agent.proxy
+  end
+
+
+  # Set PROXY agent attribute
   #
-  def self.set_headers
-    @params[:headers].each { |k, v| @request.add_field(String(k), v) } if @params[:headers]
+  # @param proxy Proxy configuration
+  #
+  def proxy= proxy
+    @agent.proxy = proxy
+  end
+
+
+  # Set BASIC_AUTH agent attribute
+  #
+  # @param username Authentification username
+  # @param password Authentification password
+  #
+  def basic_auth username, password
+    @agent.basic_auth = { username: username, password: password }
+  end
+
+
+  # Read HEADERS agent attribute
+  #
+  def headers
+    @agent.headers
+  end
+
+
+  # Set HEADERS agent attribute
+  #
+  # @param headers Request headers
+  #
+  def headers= headers
+    @agent.headers = headers
   end
 
 
-  # Make request to remote resource
+  # Read COOKIES agent attribute
   #
-  # @param uri URI object
-  # @param http
+  def cookies
+    @agent.cookies
+  end
+
+
+  # Set COOKIES agent attribute
+  #
+  # @param cookies Request cookies
   #
-  def self.get_request
-    @http.start(@uri.host, @uri.port, use_ssl: @uri.scheme == 'https') { |http| http.request(@request) }
+  def cookies= cookies
+    @agent.cookies = cookies
   end
 
+
+
+
 end

data/tiny_grabber.gemspec

@@ -20,10 +20,11 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = '~> 2.3.0'
 
-  spec.add_dependency 'socksify', '~> 1.7'
-  spec.add_dependency 'nokogiri', '~> 1.6'
+  spec.add_runtime_dependency 'socksify', '~> 1.7'
+  spec.add_runtime_dependency 'nokogiri', '~> 1.6'
+  spec.add_runtime_dependency 'redis', '~> 3.3'
 
-  spec.add_development_dependency 'bundler', '~> 1.11'
   spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'bundler', '~> 1.11'
   spec.add_development_dependency 'rspec', '~> 3.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tiny_grabber
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Aleksandr Chernyshev
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-24 00:00:00.000000000 Z
+date: 2016-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: socksify
@@ -39,19 +39,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.6'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: redis
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
-  type: :development
+        version: '3.3'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '3.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -98,9 +112,9 @@ files:
 - bin/console
 - bin/setup
 - lib/tiny_grabber.rb
+- lib/tiny_grabber/agent.rb
 - lib/tiny_grabber/http.rb
 - lib/tiny_grabber/version.rb
-- tiny_grabber-0.0.2.gem
 - tiny_grabber.gemspec
 homepage: https://github.com/moroznoeytpo/tiny_grabber
 licenses: