tweakphoeus 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ea8b67191906a4405d2a281b65bebe605b75b47a
-  data.tar.gz: f00524b1660e48dd48367f273fb59daba447cc73
+  metadata.gz: 03974c87632b69ec0956d0fb451350d8ffc5ec85
+  data.tar.gz: 42f409a675c5344ce173cc44b6d8624d333498bd
 SHA512:
-  metadata.gz: c92ea3ec329f44879363f3411f4953f8b70a5492938b42b07ebbdaa4d82a551ab8aff5b5d951106933c0402dc151bfef4c83092d15118dc92c59e49f7f807ac4
-  data.tar.gz: 22c40a0112be9ebc97f62f3a57d712e4760c448260447cf69f9be491db08b7a4b9eb82eab11f35b96416f4cf40a20f30bdabaedc995b3ee32862be8aeed4a327
+  metadata.gz: 78889b4ee6c703190e18d51996bb2b7bbf8d22a6667240283edc16b8a9f07dd0659c823d8e1910bbaff0f696300f2cb0c6510e64d334eca9db77f0efb27381f8
+  data.tar.gz: f83067fb5b6234937144eb86518277ec45e6bab6afa235d4ada784aa2645d089d83ae7e1536274dc470f5e79277d94b7ed10fd5a66ec8c45f834696f8c07af47

data/CHANGELOG.md

@@ -1 +1,2 @@
 -- v0.1.0 - First stable version: Typhoeus dependency version updated
+-- v0.2.0 - Refactor of cookies format

data/lib/tweakphoeus/version.rb

@@ -1,3 +1,3 @@
 module Tweakphoeus
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/tweakphoeus.rb

@@ -7,22 +7,24 @@ module Tweakphoeus
 
     def initialize()
       @cookie_jar = {}
+      @referer = [""]
+      @base_headers = {
+        "User-Agent" => "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:42.0) Gecko/20100101 Firefox/42.0",
+        "Accept-Language" => "es-ES,es;q=0.8,en-US;q=0.5,en;q=0.3",
+        "Accept-Encoding" => "",
+        "DNT" => "1",
+        "Connection" => "keep-alive"
+      }
     end
 
     def get(url, body: nil, headers: nil, redirect: true)
-      inject_cookies(url, headers)
-      response = Typhoeus.get(url, body: body, headers: headers)
-      obtain_cookies(response)
-      response = get(redirect_url(response), body: body, headers: headers) if redirect && has_redirect?(response)
-      response
+      set_referer_from_headers(headers)
+      http_request(url, body: body, headers: headers, redirect: redirect, method: :get)
     end
 
     def post(url, body: nil, headers: nil, redirect: false)
-      inject_cookies(url, headers)
-      response = Typhoeus.post(url, body: body, headers: headers)
-      obtain_cookies(response)
-      response = post(redirect_url(response), body: body, headers: headers) if redirect && has_redirect?(response)
-      response
+      set_referer_from_headers(headers)
+      http_request(url, body: body, headers: headers, redirect: redirect, method: :post)
     end
 
     def get_hide_inputs response
@@ -30,60 +32,106 @@ module Tweakphoeus
     end
 
     def add_cookies host, key, value
-      domain = get_domain host
-      @cookie_jar[domain] = [] if @cookie_jar[domain].nil?
-      @cookie_jar[domain] = @cookie_jar[domain].reject{|hash| hash.first[0]==key}
-      @cookie_jar[domain] << {key => value}
+      domain = get_domain(host)
+      @cookie_jar[domain] ||= {}
+      @cookie_jar[domain][key] = value
     end
 
     def get_domain domain
-      domain.match(/([a-zA-Z0-9]+:\/\/|)([^\/]+)/)[2]
+      domain.match(/([a-zA-Z0-9]+:\/\/|)([^\/]+)/)[2].gsub(/^\./,'')
+    end
+
+    def push_referer url = ""
+      @referer << url
+      url
+    end
+
+    def pop_referer
+      @referer.pop
+    end
+
+    def get_referer
+      @referer.last
     end
 
     private
 
+    def http_request(url, body: nil, headers: nil, redirect: false, method: method)
+      request_headers = @base_headers
+      request_headers["Cookie"] = inject_cookies(url, headers)
+      request_headers["Referer"] = get_referer
+      response = Typhoeus.send(method, url, body: body, headers: request_headers)
+      obtain_cookies(response)
+      set_referer(url) if method != :post
+      if redirect && has_redirect?(response)
+        if response.code != 307
+          method = :get
+          body = nil
+        end
+        response = http_request(redirect_url(response),
+                                body: body,
+                                headers: headers,
+                                redirect: redirect,
+                                method: method)
+      end
+      response
+    end
+
     def obtain_cookies response
       set_cookies_field = response.headers["Set-Cookie"]
       return if set_cookies_field.nil?
       if set_cookies_field.is_a?(String)
-        set_cookies_field = [response.headers["Set-Cookie"]]
+        set_cookies_field = [set_cookies_field]
       end
 
       set_cookies_field.each do |cookie|
-        key, value = cookie.match(/^([^=]+)=([^;].+)/).to_a[1..-1]
+        key, value = cookie.match(/^([^=]+)=(.+)/).to_a[1..-1]
         domain = cookie.match(/Domain=([^;]+)/)
 
         if domain.nil?
-          domain = get_domain response.request.url
+          domain = get_domain(response.request.url)
         else
-          domain = domain[1]
+          domain = domain[1].gsub(/^\./,'')
         end
 
+        value = value.split(';').first
         if value != "\"\""
-          @cookie_jar[domain] = [] if @cookie_jar[domain].nil?
-          @cookie_jar[domain] = @cookie_jar[domain].reject{|hash| hash.first[0]==key}
-          @cookie_jar[domain] << {key => value}
+          @cookie_jar[domain] ||= {}
+          @cookie_jar[domain][key] = value
+        else
+          @cookie_jar[domain] ||= {}
+          @cookie_jar[domain].delete(key)
         end
       end
     end
 
     def inject_cookies url, headers
-      domain = get_domain url
-      headers = {} if headers.nil?
-      cookies = []
+      domain = get_domain(url)
+      headers ||= {}
+      cookies = parse_cookie(headers["Cookie"])
 
       while domain.split(".").count > 1
         if @cookie_jar[domain]
-          @cookie_jar[domain].each do |cookie|
-            if !cookie.in?(cookies.map{|k,v| k})
-              cookies << cookie
-            end
-          end
+          cookies = @cookie_jar[domain].merge(cookies)
         end
         domain = domain.split(".")[1..-1].join(".")
       end
 
-      headers["Cookie"] = cookies.map{|hash| hash.map{|k,v| k + "=" + v}}.flatten.join('; ')
+      cookies.map{|k,v| "#{k}=#{v}"}.join('; ')
+    end
+
+    def parse_cookie cookie_string
+      cookies = {}
+
+      if cookie_string.is_a?(String)
+        cookie_string.split(';').each do |string|
+          key, value = string.split('=')
+          key.strip!
+          cookies[key.strip] = value if value && !["Domain","Path","domain","path"].include?(key)
+        end
+      end
+
+      cookies
     end
 
     def has_redirect? response
@@ -97,5 +145,16 @@ module Tweakphoeus
     def purge_bad_cookies cookies
       cookies.reject{|e| e.first.last=="\"\""}
     end
+
+    def set_referer_from_headers headers
+      if headers && headers["Referer"].is_a?(String)
+        @referer.last.replace headers["Referer"]
+      end
+    end
+
+    def set_referer url
+      @referer.last.replace url
+    end
+
   end
 end

data/tweakphoeus.gemspec

@@ -27,7 +27,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency 'typhoeus', "~> 1.0.1"
+  spec.add_dependency 'typhoeus', '~> 1.0', '>= 1.0.1'
   spec.add_development_dependency "bundler", "~> 1.10"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0", "> 3.0.0"

metadata

@@ -1,20 +1,23 @@
 --- !ruby/object:Gem::Specification
 name: tweakphoeus
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - David Martin Garcia
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-01-22 00:00:00.000000000 Z
+date: 2016-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: typhoeus
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.1
   type: :runtime
@@ -22,6 +25,9 @@ dependencies:
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.1
 - !ruby/object:Gem::Dependency
@@ -114,7 +120,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: Typhoeus on steroids.