pba_crawler 0.0.8 → 0.0.11

data/lib/pba_crawler/generic.rb

@@ -6,34 +6,64 @@ module PbaCrawler
     @@cursor = 0
     @@proxy = nil
 
+    @@proxy_list_reload = false
+
+
     def initialize
       #load proxy list
       @@proxies = Proxy.order(:position)
     end
 
-    def crawl_url(url)
-      @@proxy = @@proxies.fetch(@@cursor)
+    def crawl_url(url,use_proxy=true)
+
+      if use_proxy.eql?(true)
+
+        unless @@proxy_list_reload
+          load_proxy_list
+        end
 
-      p "#{url}|#{@@proxy.address}|#{@@proxy.port}"
-      
-      agent = Mechanize.new { |a|
-        a.html_parser = Nokogiri::HTML
-        a.open_timeout = 20
-        a.read_timeout = 20
-      }
 
-      agent.set_proxy(@@proxy.address,@@proxy.port)
-      agent.user_agent = 'Mozilla/5.0 (Windows; U; Windows NT 6.1; nl; rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13'
+        @@proxy = @@proxies.fetch(@@cursor)
 
-      get_url(agent,url)
+        p "#{url}|#{@@proxy.address}|#{@@proxy.port}"
+
+        agent = Mechanize.new { |a|
+          a.html_parser = Nokogiri::HTML
+          a.open_timeout = 20
+          a.read_timeout = 20
+        }
+
+        agent.set_proxy(@@proxy.address,@@proxy.port)
+        agent.user_agent = 'Mozilla/5.0 (Windows; U; Windows NT 6.1; nl; rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13'
+
+        get_url(agent,url)
+      else
+        agent = Mechanize.new { |a|
+          a.html_parser = Nokogiri::HTML
+          a.open_timeout = 20
+          a.read_timeout = 20
+        }
+        agent.get(url)
+      end
+    end
+
+    def next_proxy
+      if @@cursor + 1 < @@proxies.size
+        @@cursor = @@cursor + 1
+        true
+      else
+        false
+      end
     end
 
     def get_url(agent,url)
       begin
         response = agent.get(url)
         if response
+          p "ok"
           @@proxy.update_attributes(:last_request_at => Time.now)
         else
+          p "nok"
           response = rescue_get_url(agent,url)
         end
       rescue
@@ -56,6 +86,37 @@ module PbaCrawler
       @@proxy
     end
 
+    def load_proxy_list
+      #/proxylist.php?port=&type=&ssl=&country=&latency=1000&reliability=9000&sort=reliability&desc=true&pnum=1#table
+
+      url = "http://www.xroxy.com/proxylist.php?port=&type=&ssl=&country=&latency=1000&reliability=9000"
+
+      last_created_proxy = Proxy.order("created_at DESC").first
+
+      if last_created_proxy.nil? || ((Time.now - last_created_proxy.created_at) / 1.hour)>2
+        Proxy.delete_all
+        begin
+          (1..4).each do |index|
+            agent = crawl_url("#{url}&pnum=#{index}",false)
+            agent.parser.xpath("//a[@title='View this Proxy details']").each do |a|
+              address = a.content.strip
+              port = a.parent.next.next.children.first.content.strip
+              Proxy.create(:address=> address, :port=> port)
+            end
+            sleep 3
+          end
+        rescue
+
+        end
+        @@proxy_list_reload = true
+      else
+        @@proxy_list_reload = true
+      end
+      
+    end
+
+
+
     def load_sample
       list = [
         ["101.0.6.87","31280"],

data/lib/pba_crawler/version.rb

@@ -1,3 +1,3 @@
 module PbaCrawler
-  VERSION = "0.0.8"
+  VERSION = "0.0.11"
 end

metadata

@@ -2,7 +2,7 @@
 name: pba_crawler
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.0.8
+  version: 0.0.11
 platform: ruby
 authors: 
 - Pierre BASILE
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-04-19 00:00:00 +02:00
+date: 2011-05-14 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency