tiny_grabber 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 99211b6e18440ea58661cf5b5a5bdadec631f942
-  data.tar.gz: 1e78bd2089d65949431b519ae7a15b3f4bba202f
+  metadata.gz: c48375a5210123b907c6ebe2a7911f1c7090188a
+  data.tar.gz: 19b5052e7e79e876a40368d9e061a9aba032b7b4
 SHA512:
-  metadata.gz: 3a4dc8a114ea9818dee2bd400ccc5e45c60ae40f2a9e7ce5002c51b214b9e471d1066f2d865e48ab0acbe91e80c51d55cb17ed43a7eed29e01283f82c61c09b8
-  data.tar.gz: 4ec0f357261f447f3cd34dda16a9376dd770a73a9c7fc30b22577f42de1b4a33bd557b5c8da85ce54a5d40d780af4e152ebbd93b0d50097fbf0c7081d134f30e
+  metadata.gz: 498cffe418cabb9917dad706d8df5d6281fcc119418016dcd775169346fd5108fcdbf0af5c8d75cbb1472acfdef96d4bc82e037cfc339de729fd864f1dcdafc2
+  data.tar.gz: 582d3bf2fa5cde9a54c2862d5f5d237218d483f69dadd8749eba93a07ab58a17ecfd1d40aaac5197823a155363bd51ae1196f5de033c33239432cd7b76d281b6

data/README.md

@@ -28,7 +28,7 @@ Or install it yourself as:
 require 'tiny_grabber'
 
 read_timeout = 300
-user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.94 Safari/537.36'
+user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36'
 proxy = { ip: 'xx.xx.xx.xx', port: 'xxxx' }
 headers1 = { 'Content-Type' => 'text/html; charset=utf-8' }
 headers2 = { 'Content-Type' => 'text/html; charset=utf-8', 'Connection' => 'keep-alive' }
@@ -54,13 +54,16 @@ tg.cookies = cookies
 
 # Make response with GET method
 response = tg.get 'https://whoer.net/ru', headers
+# Reset headers and cookies
+tg.reset
 # Make response with POST method
-response = tg.get 'https://whoer.net/ru', params, headers
+response = tg.post 'https://whoer.net/ru', params, headers
+
 
 # Make singleton response with GET method
-response = TinyGrabber.get 'https://whoer.net/ru', headers, { debug = true, read_timeout = read_timeout ...}
+response = TinyGrabber.get 'https://whoer.net/ru', { debug = true, read_timeout = read_timeout ... }
 # Make singleton response with POST method
-response = TinyGrabber.post 'https://whoer.net/ru', params, headers, { debug = true, read_timeout = read_timeout ...}
+response = TinyGrabber.post 'https://whoer.net/ru', params, { debug = true, read_timeout = read_timeout ... }
 
 # Get Nokogiri object from response HTML
 ng = response.ng
@@ -70,11 +73,18 @@ response.code
 response.cookies
 # Get response headers
 response.headers
-
+# Get response HTML
+response.body
 ```
 
 ## Changelog
 
+* *v 0.2.1*
+    * Setting random user_agent from list if it not seted
+    * Remove headers attribute from singleton methods
+    * Remove header transfer-encoding for chain requests
+    * Add reset method for delete headers and cookies
+
 * *v 0.2.0*
     * Now there is an opportunity to create object TinyGrabber
     * Change order of parameters for singleton request

data/lib/tiny_grabber.rb

@@ -47,9 +47,9 @@ class TinyGrabber
   # @param url Resource link
   # @param headers Request header
   #
-  def self.get url, headers = {}, config = {}
+  def self.get url, config = {}
     initialize  config
-    @agent.fetch url, :get, headers
+    @agent.fetch url, :get
   end
 
 
@@ -59,7 +59,7 @@ class TinyGrabber
   # @param params Request post data
   # @param headers Request header
   #
-  def post url, params = {}, headers = {}, config = {}
+  def post url, params = {}, headers = {}
     @agent.fetch url, :post, headers, params
   end
 
@@ -69,9 +69,9 @@ class TinyGrabber
   # @param url Resource link
   # @param headers Request header
   #
-  def self.post url, params = {}, headers = {}, config = {}
+  def self.post url, params = {}, config = {}
     initialize  config
-    @agent.fetch url, :post, headers, params
+    @agent.fetch url, :post, {}, params
   end
 
 
@@ -173,7 +173,9 @@ class TinyGrabber
     @agent.cookies = cookies
   end
 
-
-
-
+  # Call RESET agent method
+  #
+  def reset
+    @agent.reset
+  end
 end

data/lib/tiny_grabber/agent.rb

@@ -18,13 +18,37 @@ class TinyGrabber::Agent
   # Headers
   attr_accessor :cookies
 
+  #  Agent aliases given from http://www.useragentstring.com/pages/Chrome/
+  AGENT_ALIASES = [
+    # Chrome
+    'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36',
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36',
+    # Firefox
+    'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1',
+    'Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:33.0) Gecko/20100101 Firefox/33.0',
+    'Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/31.0',
+    # Internet Explorer
+    'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko',
+    'Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0',
+    'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 7.0; InfoPath.3; .NET CLR 3.1.40767; Trident/6.0; en-IN)',
+    'Mozilla/5.0 (compatible; MSIE 10.0; Macintosh; Intel Mac OS X 10_7_3; Trident/6.0)',
+    # Opera
+    'Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16',
+    'Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14',
+    'Mozilla/5.0 (Windows NT 6.0; rv:2.0) Gecko/20100101 Firefox/4.0 Opera 12.14',
+    'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52',
+  ]
+
   # Initialization object
   #
   def initialize
     @debug = false
 
     # Initialize variables agent attributes
-    @user_agent = nil
+    @user_agent = AGENT_ALIASES[rand(AGENT_ALIASES.count) - 1]
     @proxy = []
     @basic_auth = {}
     @headers = {}
@@ -126,7 +150,7 @@ class TinyGrabber::Agent
     if @debug
       p "#{debug_initial_word} =============================="
       p "#{debug_initial_word} #{method.upcase} #{url}"
-      p "#{debug_initial_word} #{params}"
+      p "#{debug_initial_word} -> [params] = #{params}"
       p "#{debug_initial_word} ------------------------------"
     end
     set_uri url
@@ -160,7 +184,7 @@ class TinyGrabber::Agent
     if @debug
       debug_filename = "log/#{method.upcase}_#{@uri.to_s.gsub(/[\/:]/, '_').gsub(/_+/, '_')}"
       File.open(debug_filename, 'wb') { |f| f << @response.body } if @debug
-      p "#{debug_initial_word} HTML > #{debug_filename}"
+      p "#{debug_initial_word} <- [html_file] = #{debug_filename}"
     end
     @response
   end
@@ -173,7 +197,7 @@ class TinyGrabber::Agent
   def set_uri url
     # It's magic work with escaped url
     @uri = URI(URI.escape(URI.unescape(url)))
-    p "#{debug_initial_word} URI = #{@uri}" if @debug
+    p "#{debug_initial_word} -> [uri] = #{@uri}" if @debug
   end
 
 
@@ -181,7 +205,7 @@ class TinyGrabber::Agent
   #
   def set_user_agent
     @headers['User-Agent'] = @user_agent
-    p "#{debug_initial_word} user_agent = #{@user_agent}" if @debug
+    p "#{debug_initial_word} -> [user_agent] = #{@user_agent}" if @debug
   end
 
 
@@ -189,7 +213,7 @@ class TinyGrabber::Agent
   #
   def set_basic_auth
     @request.basic_auth @basic_auth[:username], @basic_auth[:password]
-    p "#{debug_initial_word} basic_auth = #{@basic_auth}" if @debug
+    p "#{debug_initial_word} -> [basic_auth] = #{@basic_auth}" if @debug
   end
 
 
@@ -197,7 +221,7 @@ class TinyGrabber::Agent
   #
   def set_headers
     @headers.each { |k, v| @request.add_field(String(k), v) }
-    p "#{debug_initial_word} headers = #{@headers}" if @debug
+    p "#{debug_initial_word} -> [headers] = #{@headers}" if @debug
   end
 
 
@@ -205,7 +229,7 @@ class TinyGrabber::Agent
   #
   def set_cookies
     @request['Cookie'] = @cookies
-    p "#{debug_initial_word} cookies = #{@cookies}" if @debug
+    p "#{debug_initial_word} -> [cookies] = #{@cookies}" if @debug
   end
 
 
@@ -215,7 +239,7 @@ class TinyGrabber::Agent
   def send_request
     @http.start(@uri.host, @uri.port, use_ssl: @uri.scheme == 'https') do |http|
       http.read_timeout = @read_timeout
-      p "#{debug_initial_word} read_timeout = #{@read_timeout}" if @debug
+      p "#{debug_initial_word} -> [read_timeout] = #{@read_timeout}" if @debug
       http.request(@request)
     end
   end
@@ -225,7 +249,9 @@ class TinyGrabber::Agent
   #
   def save_headers
     @headers = @response.headers
-    p "#{debug_initial_word} save_headers = #{@headers}" if @debug
+    # Delete header TRANSFER_ENCODING for chain of requests
+    @headers.delete('transfer-encoding')
+    p "#{debug_initial_word} <- [headers] = #{@headers}" if @debug
   end
 
 
@@ -233,7 +259,15 @@ class TinyGrabber::Agent
   #
   def save_cookies
     @cookies = @response.cookies
-    p "#{debug_initial_word} save_cookies = #{@cookies}" if @debug
+    p "#{debug_initial_word} <- [cookies] = #{@cookies}" if @debug
+  end
+
+
+  # Clears headers and cookies
+  #
+  def reset
+    @headers = {}
+    @cookies = nil
   end
 
 

data/lib/tiny_grabber/version.rb

@@ -1,4 +1,4 @@
 class TinyGrabber
   # Version number
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tiny_grabber
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Aleksandr Chernyshev
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-02 00:00:00.000000000 Z
+date: 2016-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: socksify