the_mask 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2e23912c7fb5129bb925acbd4e9e609ebe056919
-  data.tar.gz: 552488893034e2a9833f794c8d09af9e9b246531
+  metadata.gz: d67508982b071a8217ee5ca318f3ad041dc601d1
+  data.tar.gz: c29f73ee46620975f3671caf575a9c34726a1047
 SHA512:
-  metadata.gz: cf1ecb0d2bf67090fc760a5bb5f198127a3050e6c442c62ddb50a2c3d99c66f59abc177152d58bc082c59ead384087a3d6b620ec648bc14670b9ace58376fb63
-  data.tar.gz: c5459ce1dd5e26f70b3e7f7202294bf99339ec48df2d6671d7f4edb58bd101c720def4dd7d043e5f6032210e96ed4ad593fa316a6963359934c4e73738bb127e
+  metadata.gz: d6181d81ef0efb3c02f36f70a4731bf8f55ae62e186ba09f3cbd996300e2a288ac0af2568ae1df0746b109866dd3580fa7de09c666e3ef9a08c0a5576ebc8266
+  data.tar.gz: e6b3d53f7d9a8465d825e0273194ac89e55299f5443a7e33e91d674fb48fadc69c2f7f9543a7bd145a98bd739988964752c07efac847530ed8bc98693e6f2259

data/README.md

@@ -4,6 +4,13 @@
 
 Tired of issues involved with data mining? Put on The Mask and try data mining designed for the next generation.
 
+## Features
+
+- A powerful internal proxy list manager that prioritizes successful mining and retrieval of data.  
+- Full control over the data mining process through various configurations available (see below for options).
+- Socket obfuscation (in-progress)
+
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -12,36 +19,40 @@ Add this line to your application's Gemfile:
 gem 'the_mask'
 ```
 
-And then execute:
-
-    $ bundle
-
 Or install it yourself as:
 
     $ gem install the_mask
 
 ## Usage
 
-`mask_connect = TheMask::Connect.new(read_timeout: 4, open_timeout: 4, max_tries: 4)
- mask_connect.open_url 'http://www.abcdefg.com'
-`
+```ruby
+mask_connect = TheMask::Connect.new(read_timeout: 4, open_timeout: 4, max_tries: 4)
+mask_connect.open_url('http://www.abcdefg.com')
+```
   
 This will return the body data from the supplied URL.  
   
 Available options:  
-`read_timeout = Read timeout in seconds (default: 3)`  
-`open_timeout = Open timeout in seconds (default: 3)`  
-`timeout = Timeout for whole procedure in seconds (default: 5)`  
-`max_tries = Maximum attempts in reading the page (default: 3)`  
-`min_page_length = Minimum page length in bytes, if not satisfied, reattempt retrieval (default: 100 bytes)`   
-`reset_ua = Reset user agent on every request. (default: true)`  
-`force = Force continuous opening of page until data is retrieved (default: false)`  
+```
+read_timeout = Read timeout in seconds (default: 3)
+open_timeout = Open timeout in seconds (default: 3)
+timeout = Timeout for whole procedure in seconds (default: 5)
+max_tries = Maximum attempts in reading the page (default: 3)
+min_page_length = Minimum page length in bytes, if not satisfied, reattempt retrieval (default: 100 bytes)
+reset_ua = Reset user agent on every request. (default: true)
+force = Force continuous opening of page until data is retrieved (default: false)
+min_proxy_response_time = Minimum response time for proxies in seconds. After executing open_url, if proxy response time is over set minimum, proxy will be removed from internal proxy list (default: no minimum response time)
+```
 
 Proxy options example:  
-`mask_connect = TheMask::Connect.new(proxy: { ip: '127.0.0.1', port: 8080, username: 'asd333', password: 'asd333' })`  
+```ruby
+mask_connect = TheMask::Connect.new(proxy: { ip: '127.0.0.1', port: 8080, username: 'asd333', password: 'asd333' })
+```
   
 Or supply multiple proxies with an array:  
-`mask_connect = TheMask::Connect.new(proxies: ['111.11.1.1:80', '10.10.101.10:800', '192.10.10.1:80:sdad:asdasd'])`  
+```ruby
+mask_connect = TheMask::Connect.new(proxies: ['111.11.1.1:80', '10.10.101.10:800', '192.10.10.1:80:sdad:asdasd'])
+```  
 
 
 ## Development

data/lib/the_mask/socket.rb

@@ -8,6 +8,7 @@ module TheMask
     MINIMUM_PAGE_LENGTH = 100 #bytes
     FORCE_READ = false
     RESET_USER_AGENT = true
+    MIN_PROXY_RESPONSE_TIME = nil #seconds, default: nil = do not remove proxies
 
     def initialize(options = {})
       @proxies = nil
@@ -16,6 +17,7 @@ module TheMask
       @force = options[:force] || FORCE_READ
       @min_page_length = options[:min_page_length] || MINIMUM_PAGE_LENGTH
       @reset_user_agent = options[:reset_ua] || RESET_USER_AGENT
+      @min_proxy_response_time = options[:min_proxy_response_time] || MIN_PROXY_RESPONSE_TIME
 
       @agent = Mechanize.new
 
@@ -39,9 +41,14 @@ module TheMask
 
     def open_url(url)
       read_proc = Proc.new do
+        proxy = nil #Selected proxy
         tries = 0 #Total URL retrieval tries
         page_data = nil #Retrieved page html data
 
+        #Variables for timing the GET request
+        end_time = nil
+        start_time = nil
+
         begin
           tries += 1
 
@@ -51,8 +58,6 @@ module TheMask
 
           @agent.user_agent = TheMask.get_random_user_agent_str if @reset_user_agent
 
-          proxy = nil
-
           begin
             unless @proxies.nil?
               begin
@@ -72,7 +77,9 @@ module TheMask
           end
 
           Timeout::timeout(@timeout) do
+            start_time = Time.now
             page_data = @agent.get url
+            end_time = Time.now
           end
         rescue Errno::ETIMEDOUT => e
           retry
@@ -95,6 +102,13 @@ module TheMask
         rescue
           retry
         end
+
+        unless @min_proxy_response_time.nil? || start_time.nil? || end_time.nil?
+          #Remove proxy from list if response time is longer than the minimum response time provided in options
+          response_time = end_time - start_time
+          @proxies.remove_proxy!(proxy) if response_time > @min_proxy_response_time
+        end
+
         page_data
       end
 

data/lib/the_mask/version.rb

@@ -1,3 +1,3 @@
 module TheMask
-  VERSION = "0.2.1"
+  VERSION = "0.2.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: the_mask
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Saoud Khalifah
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-11-19 00:00:00.000000000 Z
+date: 2015-11-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler