requestmanager 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3f659f37925e9e2719610b340be45956537bd03b
-  data.tar.gz: 42a0ec24a2c767b540494f2ee61c1cbf59a777f8
+  metadata.gz: 48391942056d9ec893ed2e68d7ea93e07d1095e1
+  data.tar.gz: ac26cce67927dcd1e9fad3ca2991852d48f97d7b
 SHA512:
-  metadata.gz: 6b5207c4531a5dd3b717d5e12ce37d3c9e41296c156a2d40a2249c0baf46b4cf0e35d5f328783917868bcd5f8f9ba11585e38ab4001b290a67c5f802af7d30d0
-  data.tar.gz: c748d9ab381d76ce132f51eb317829d23cd7d51267c9b273ff25386c4adb0660bb473156bbc5bddb1c6dfe8542cd4c663b04dde88b53f6fa4ad02e531d01e21a
+  metadata.gz: 63c284f54b0ce6e223918d6f054c9727991ca46d12ef4bd7dfc8fd65a52bc50897449348c261ed4976420c5a19b6b0c285b4d755860479b7943b1b6584850004
+  data.tar.gz: a2013b4535a16bd29a88aa650152106a5e5b8e73b73a015bf4b58950316a39e78d8a0e98bb9293db20b25952315e0e21c2aa2c0f08c2a2862ec36722b81e8d69

data/lib/requestmanager.rb

@@ -4,17 +4,78 @@ require 'pry'
 
 
 class RequestManager
-  def initialize(proxy_list, request_interval)
+  def initialize(proxy_list, request_interval, browser_num)
     @proxy_list = parse_proxy_list(proxy_list)
     @request_interval = request_interval
-    @used_proxies = Hash.new
+    @used_proxies = Array.new
+    @browser_num = browser_num
+    @browsers = Hash.new
+    open_n_browsers
+  end
+
+  # Open the specified number of browsers
+  def open_n_browsers
+    (1..@browser_num).each do |i|
+      open_browser
+    end
+  end
+
+  # Open the browser with a random proxy
+  def open_browser
+    chosen_proxy = @proxy_list != nil ? get_random_proxy : nil
+    @browsers[chosen_proxy] = [gen_driver(chosen_proxy), Time.now]
+  end
+
+  # Get the most recently used browser
+  def get_most_recent_browser
+    most_recent = @browsers.first
+    @browsers.each do |browser|
+      if browser[1][1] > most_recent[1][1]
+        most_recent = browser
+      end
+    end
+
+    return most_recent
+  end
+
+  # Get the least recently used browser
+  def get_least_recent_browser
+    least_recent = @browsers.first
+    @browsers.each do |browser|
+        if browser[1][1] < least_recent[1][1]
+          least_recent = browser
+        end
+    end
+    
+    # Update the usage time
+    @browsers[least_recent[0]] = [least_recent[1][0], Time.now]
+    return least_recent[1][0]
+  end
+
+  # Restart the browser and open new one
+  def restart_browser
+    # Get most recently used browser and close it
+    close_browser = get_most_recent_browser
+    close_browser[1][0].quit
+
+    # Remove it from lists of used browsers and start new
+    @browsers.delete(close_browser[0])
+    open_browser
+    @used_proxies.delete(close_browser[0])
+  end
+
+  # Close all the browsers
+  def close_all_browsers
+    @browsers.each do |browser|
+      browser[1][0].quit
+    end
   end
 
   # Get the page requested
   def get_page(url, form_input = nil)
-    chosen_proxy = @proxy_list != nil ? get_random_proxy(url) : nil
-    driver = gen_driver(chosen_proxy)
-    driver.navigate.to url
+    # Get the page
+    browser = get_least_recent_browser
+    browser.navigate.to url
     puts "Getting page " + url
 
     # Handle form input if there is any
@@ -24,10 +85,9 @@ class RequestManager
       element.submit
     end
 
-    # Sleep while things load then save
-    sleep(7)
-    page_html = driver.page_source
-    driver.quit
+    # Sleep while things load then save output
+    sleep(rand(@request_interval[0]..@request_interval[1]))
+    page_html = browser.page_source
     return page_html
   end
   
@@ -49,27 +109,21 @@ class RequestManager
   end
 
   # Choose a random proxy that hasn't been used recently
-  def get_random_proxy(url)
+  def get_random_proxy
     max = @proxy_list.length
     chosen = @proxy_list[Random.rand(max)]
-
+    chosen_proxy = chosen[0]+":"+chosen[1]
+    
     # Only use proxy if it hasn't been used in last n seconds on same host
-    if is_not_used?(chosen, url)
-      @used_proxies[chosen] = [Time.now, URI.parse(url).host]
-      return chosen[0]+":"+chosen[1]
+    if !@used_proxies.include?(chosen_proxy)
+      @used_proxies.push(chosen_proxy)
+      return chosen_proxy
     else
       sleep(0.005)
-      get_random_proxy(url)
+      get_random_proxy
     end
   end
 
-  # Checks if a proxy has been used on domain in the last 20 seconds
-  def is_not_used?(chosen, url)
-    return (!@used_proxies[chosen] ||
-            @used_proxies[chosen][0] <= Time.now-@request_interval[0] ||
-            @used_proxies[chosen][1] != URI.parse(url).host)
-  end
-
   # Parse the proxy list
   def parse_proxy_list(proxy_file)
     if proxy_file

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: requestmanager
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - M. C. McGrath
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-01 00:00:00.000000000 Z
+date: 2015-11-03 00:00:00.000000000 Z
 dependencies: []
 description: Manages proxies, wait intervals, etc
 email: shidash@shidash.com