the_mask 0.2.2 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d67508982b071a8217ee5ca318f3ad041dc601d1
-  data.tar.gz: c29f73ee46620975f3671caf575a9c34726a1047
+  metadata.gz: 81556b7f9b9fb2c1bdf439166e30b19c53fefb31
+  data.tar.gz: 8a63e7102e20188ca209d1141524891241b963b9
 SHA512:
-  metadata.gz: d6181d81ef0efb3c02f36f70a4731bf8f55ae62e186ba09f3cbd996300e2a288ac0af2568ae1df0746b109866dd3580fa7de09c666e3ef9a08c0a5576ebc8266
-  data.tar.gz: e6b3d53f7d9a8465d825e0273194ac89e55299f5443a7e33e91d674fb48fadc69c2f7f9543a7bd145a98bd739988964752c07efac847530ed8bc98693e6f2259
+  metadata.gz: 43edf63a4c770bc64852dd9c7bd832792b7fc3e051ff8d4f70e4a1e908e7aa7f3e532b7df666869de15f9b59135bc159558767120ddadf9e250328a6d08077aa
+  data.tar.gz: 3d4918317db7ff14b716818d16bddf0c73585aa70a1d98fc97af2f91f3858168426427408f4fd0214e3d40e1a17700980479fec13cae84db088b3665919425ba

data/Gemfile

@@ -2,3 +2,5 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in the_mask.gemspec
 gemspec
+
+gem 'socksify', git: 'git@github.com:saouddk/socksify-ruby.git'

data/lib/the_mask.rb

@@ -7,5 +7,6 @@ end
 
 require 'the_mask/user_agents'
 require 'the_mask/proxy_list'
+require 'the_mask/mechanize_socks_support'
 require 'the_mask/socket'
 require 'the_mask/connect'

data/lib/the_mask/mechanize_socks_support.rb

@@ -0,0 +1,38 @@
+require 'socksify'
+require 'socksify/http'
+
+module MechanizeSOCKSSupport
+  def set_proxy(addr, port, user = nil, pass = nil)
+    @socks = false
+    super
+  end
+
+  def fetch(uri, method = :get, headers = {}, params = [], referer = current_page, redirects = 0)
+    if @socks && !@socks_http.nil?
+      html = @socks_http.get URI(uri)
+      page = Struct.new(:uri, :body)
+      page.new(uri, html)
+    else
+      super
+    end
+  end
+
+  class Mechanize::HTTP::Agent
+    prepend MechanizeSOCKSSupport
+    attr_accessor :http, :socks
+
+    public
+      def set_socks(addr, port)
+        set_http unless @http
+        @socks_http = Net::HTTP::SOCKSProxy(addr, port)
+        @socks = true
+      end
+
+    private
+      def set_http
+        @http = Net::HTTP::Persistent.new 'mechanize'
+        @http.idle_timeout = 5
+        @http.keep_alive   = 300
+      end
+  end
+end

data/lib/the_mask/proxy_list.rb

@@ -1,13 +1,30 @@
 module TheMask
   class ProxyList
-    #TheMask::ProxyList::Proxy class
     class Proxy
-      attr_accessor :ip, :port, :username, :password
+      attr_accessor :ip, :port, :username, :password, :type, :socks_version
+      HTTP_PROXY = :http
+      SOCKS_PROXY = :socks
+      SUPPORTED_SOCKS_VERSIONS = ['4', '5']
 
-      def initialize(string = '')
-        unless string.empty?
+      def initialize(input_string = '')
+        unless input_string.empty?
           # Proxy string format = ip:port:username:password
-          split_str = string.split ":"
+          split_str = input_string.split(':')
+          last_element = split_str[-1].to_s.downcase
+
+          # Check if proxy has SOCKS parameter enabled, by default a proxy will always be a HTTP/s proxy
+          if last_element.eql?(SOCKS_PROXY.to_s) && (split_str.length == 3 || split_str.length == 5)
+            @type = SOCKS_PROXY
+            socks_version = last_element[-1]
+            if SUPPORTED_SOCKS_VERSIONS.include?(socks_version)
+              @socks_version = socks_version.to_i
+            else
+              # Default version is the latest SOCKS, in this case ver. 5s
+              @socks_version = SUPPORTED_SOCKS_VERSIONS[-1].to_i
+            end
+          else
+            @type = HTTP_PROXY
+          end
           @ip = split_str[0].to_s
           @port = split_str[1].to_i
           @username = split_str[2].to_s unless split_str[2].nil?
@@ -20,8 +37,14 @@ module TheMask
         end
       end
 
+      def is_SOCKS?
+        @type == SOCKS_PROXY
+      end
+
+      def is_HTTP?
+        @type == HTTP_PROXY
+      end
     end
-    #ProxyList class
     attr_accessor :proxy_list
 
     def initialize(arr = [])

data/lib/the_mask/socket.rb

@@ -1,17 +1,22 @@
+require_relative 'mechanize_socks_support'
+
 module TheMask
+  include MechanizeSOCKSSupport
+
   class Socket
-    #TODO: Move from Mechanize to native Sockets ;)
-    DEFAULT_OPEN_TIMEOUT = 3 #seconds
-    DEFAULT_READ_TIMEOUT = 3 #seconds
-    GENERAL_TIMEOUT = 5 #seconds
+    DEFAULT_OPEN_TIMEOUT = 3 # seconds
+    DEFAULT_READ_TIMEOUT = 3 # seconds
+    GENERAL_TIMEOUT = 5 # seconds
     MAXIMUM_TRIES = 3
-    MINIMUM_PAGE_LENGTH = 100 #bytes
+    MINIMUM_PAGE_LENGTH = 100 # bytes
     FORCE_READ = false
     RESET_USER_AGENT = true
-    MIN_PROXY_RESPONSE_TIME = nil #seconds, default: nil = do not remove proxies
+    MIN_PROXY_RESPONSE_TIME = nil # seconds, default: nil = do not remove proxies
+    SOCKS_INCREASE_TIMEOUTS = 0.5 # increase timeout duration by specified magnitude if proxy used for retrieval is SOCKS4/5
 
     def initialize(options = {})
       @proxies = nil
+      @socks_increase_timeouts = options[:socks_increase_timeouts] || SOCKS_INCREASE_TIMEOUTS
       @timeout = options[:timeout] || GENERAL_TIMEOUT
       @max_tries = options[:max_tries] || MAXIMUM_TRIES
       @force = options[:force] || FORCE_READ
@@ -20,20 +25,15 @@ module TheMask
       @min_proxy_response_time = options[:min_proxy_response_time] || MIN_PROXY_RESPONSE_TIME
 
       @agent = Mechanize.new
+      @agent.history.max_size = 0
 
-      @agent.open_timeout = options[:open_timeout] || DEFAULT_OPEN_TIMEOUT
-      @agent.read_timeout = options[:read_timeout] || DEFAULT_READ_TIMEOUT
+      @open_timeout = options[:open_timeout] || DEFAULT_OPEN_TIMEOUT
+      @read_timeout = options[:read_timeout] || DEFAULT_READ_TIMEOUT
 
-      unless options[:proxies]
-        if options[:proxy]
-          if options[:proxy][:username] &&  options[:proxy][:password]
-            @agent.set_proxy options[:proxy][:ip], options[:proxy][:port], options[:proxy][:username], options[:proxy][:password]
-          else
-            @agent.set_proxy options[:proxy][:ip], options[:proxy][:port]
-          end
-        end
-      else
+      if options[:proxies]
         @proxies = TheMask::ProxyList.new(options[:proxies])
+      else
+        @proxies = TheMask::ProxyList.new([options[:proxy]])
       end
 
       @agent.user_agent = TheMask.get_random_user_agent_str unless @reset_user_agent
@@ -41,11 +41,12 @@ module TheMask
 
     def open_url(url)
       read_proc = Proc.new do
-        proxy = nil #Selected proxy
-        tries = 0 #Total URL retrieval tries
-        page_data = nil #Retrieved page html data
+        proxy = nil # Selected proxy
+        tries = 0 # Total URL retrieval tries
+        page_data = nil # Retrieved page html data
+        timeout_adjustments = nil # Adjustments to timeouts based on proxy type
 
-        #Variables for timing the GET request
+        # Variables for timing the GET request
         end_time = nil
         start_time = nil
 
@@ -63,30 +64,42 @@ module TheMask
               begin
                 proxy = @proxies.get_proxy
 
-                if proxy.username && proxy.password
-                  @agent.set_proxy proxy.ip, proxy.port, proxy.username, proxy.password
+                if proxy.is_SOCKS?
+                  @agent.agent.set_socks proxy.ip, proxy.port
+                  timeout_adjustments = calculate_timeouts @socks_increase_timeouts
+                elsif proxy.is_HTTP?
+                  if proxy.username && proxy.password
+                    @agent.set_proxy proxy.ip, proxy.port, proxy.username, proxy.password
+                  else
+                    @agent.set_proxy proxy.ip, proxy.port
+                  end
+                  timeout_adjustments = calculate_timeouts
                 else
-                  @agent.set_proxy proxy.ip, proxy.port
+                  raise "TheMask: unknown proxy type '#{proxy.type}'."
                 end
               end
             end
           rescue Timeout::ExitException => e
-            #Exception timeout from mechanize
+            # Exception timeout from mechanize
             @proxies.remove_proxy!(proxy)
             retry
           end
-
-          Timeout::timeout(@timeout) do
+          @agent.open_timeout = timeout_adjustments[:open_timeout]
+          @agent.read_timeout = timeout_adjustments[:read_timeout]
+          Timeout::timeout(timeout_adjustments[:timeout]) do
             start_time = Time.now
             page_data = @agent.get url
             end_time = Time.now
           end
+
         rescue Errno::ETIMEDOUT => e
           retry
         rescue Net::HTTP::Persistent::Error => e
           retry
         rescue Timeout::Error => e
           retry
+        rescue SOCKSError => e
+          retry
         rescue SignalException => e
           retry
         rescue Net::HTTPNotFound => e
@@ -104,7 +117,7 @@ module TheMask
         end
 
         unless @min_proxy_response_time.nil? || start_time.nil? || end_time.nil?
-          #Remove proxy from list if response time is longer than the minimum response time provided in options
+          # Remove proxy from list if response time is longer than the minimum response time provided in options
           response_time = end_time - start_time
           @proxies.remove_proxy!(proxy) if response_time > @min_proxy_response_time
         end
@@ -115,7 +128,6 @@ module TheMask
       if @force
         while true
           data = read_proc.call
-
           unless data.nil? || data.body.to_s.empty? || data.body.to_s.length < @min_page_length
             return data.body
           end
@@ -124,5 +136,14 @@ module TheMask
 
       read_proc.call.body
     end
+
+    private
+      def calculate_timeouts(increase_magnitude = 0)
+        {
+            timeout: (@timeout + (@timeout * increase_magnitude)),
+            open_timeout: (@open_timeout + (@open_timeout * increase_magnitude)),
+            read_timeout: (@read_timeout + (@read_timeout * increase_magnitude))
+        }
+      end
   end
 end

data/lib/the_mask/version.rb

@@ -1,3 +1,3 @@
 module TheMask
-  VERSION = "0.2.2"
+  VERSION = "0.2.5"
 end

data/the_mask.gemspec

@@ -23,5 +23,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 0'
 
-  spec.add_dependency 'mechanize', '~> 2.7'
+  spec.add_dependency 'mechanize', '~> 2.7.4'
+  spec.add_dependency 'socksify', '~> 1.7.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: the_mask
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.5
 platform: ruby
 authors:
 - Saoud Khalifah
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-11-29 00:00:00.000000000 Z
+date: 2016-06-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -58,14 +58,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: 2.7.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: 2.7.4
+- !ruby/object:Gem::Dependency
+  name: socksify
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.0
 description: TheMask provides functionality for the purposes of data mining.
 email:
 - saouddk@gmail.com
@@ -85,6 +99,7 @@ files:
 - bin/setup
 - lib/the_mask.rb
 - lib/the_mask/connect.rb
+- lib/the_mask/mechanize_socks_support.rb
 - lib/the_mask/proxy_list.rb
 - lib/the_mask/socket.rb
 - lib/the_mask/user_agents.rb
@@ -115,3 +130,4 @@ signing_key:
 specification_version: 4
 summary: Socket obfuscation for purposes of data mining/gathering.
 test_files: []
+has_rdoc: