list_spider 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 45ea1dba6db98ca7a9cdaecde7f744728cd20b03
-  data.tar.gz: 118764345cebb58a37e15af591b3f007451c2486
+  metadata.gz: facd4c45a64f0cca934e70de9cfda4f05cf1a1d9
+  data.tar.gz: b0e5940faa51be59c13a0efcfc114b43981082b7
 SHA512:
-  metadata.gz: 673150361b67fd16cf7dc86560c0bbe17d3d432f3f40dc4456019e9700d0d68f3b1d9eea8d6c036fc3ea904866497d248b51a36007e345a9233a43b827d0846b
-  data.tar.gz: 5c2b99885733c979d9e1f9f2426521b125fce8dd951a3f51c96d25c33ae1c180b0aeb70654b5b4422b0691bb337fdd517834cb28fa0edbee2798e895c6aa2465
+  metadata.gz: a21daf8996b6aa27714a6407511a094f6381f739a65066077ca89bb0f4783ba0e6576b48080dbdaacee73d1a2ea4e6c77195ba832ee8f0bc28bde295f8e50b8c
+  data.tar.gz: e5bb44f131f82f480d9b5ebd146587c6473f6d3a7433fea32dd46bb74ac2c76adebc362048ee1c6228ca563ae4aa3d1b69bcc6d8735a4ee683e5c848f60944a4

data/lib/list_spider.rb

@@ -66,89 +66,98 @@ module ListSpider
       failed_list = []
       succeed_list = []
       multi = EventMachine::MultiRequest.new
-      # no_job = true
       begin_time = Time.now
 
       for_each_proc = proc do |e|
-        # if !@@overwrite_exist && File.exist?(e.local_path)
-        #   succeed_list << e
-        # else
-          next unless @@url_set.add?(e.href)
-          # no_job = false
-          opt = {}
-          opt = {:redirects => @@max_redirects}
-          opt[:head] = @@header_option if defined? @@header_option
-          if e.http_method == :post
-            opt[:body] = e.params unless e.params.empty?
-            if @@connection_opts
-              w = EventMachine::HttpRequest.new(e.href, @@connection_opts).post opt
-            else
-              w = EventMachine::HttpRequest.new(e.href).post opt
-            end
+        opt = {}
+        opt = {:redirects => @@max_redirects}
+        opt[:head] = @@header_option if defined? @@header_option
+        if e.http_method == :post
+          opt[:body] = e.params unless e.params.empty?
+          if @@connection_opts
+            w = EventMachine::HttpRequest.new(e.href, @@connection_opts).post opt
           else
-            if @@connection_opts
-              opt[:query] = e.params unless e.params.empty?
-              w = EventMachine::HttpRequest.new(e.href, @@connection_opts).get opt
-            else
-              w = EventMachine::HttpRequest.new(e.href).get opt
-            end
+            w = EventMachine::HttpRequest.new(e.href).post opt
+          end
+        else
+          if @@connection_opts
+            opt[:query] = e.params unless e.params.empty?
+            w = EventMachine::HttpRequest.new(e.href, @@connection_opts).get opt
+          else
+            w = EventMachine::HttpRequest.new(e.href).get opt
           end
+        end
 
-          w.callback {
-            @@url_set.delete(e.href)
-            # puts "complete:#{w.response_header}"
-            s = w.response_header.status
-            puts s
-            if s != 404
-              local_dir = File.dirname(e.local_path)
-              FileUtils.mkdir_p(local_dir) unless Dir.exist?(local_dir)
-              begin
-                File.open(e.local_path, "w") do |f|
-                  if @@conver_to_utf8 == true
-                    f << SpiderHelper.to_utf8( w.response)
-                  else
-                    f << w.response
-                  end
+        w.callback {
+          s = w.response_header.status
+          puts s
+          if s != 404
+            local_dir = File.dirname(e.local_path)
+            FileUtils.mkdir_p(local_dir) unless Dir.exist?(local_dir)
+            begin
+              File.open(e.local_path, "w") do |f|
+                if @@conver_to_utf8 == true
+                  f << SpiderHelper.to_utf8( w.response)
+                else
+                  f << w.response
                 end
-                succeed_list << e
-              rescue Exception => e
-                puts e
               end
+              succeed_list << e
+            rescue Exception => e
+              puts e
             end
-          }
-          w.errback {
-            @@url_set.delete(e.href)
-            puts "errback:#{w.response_header}"
-            puts e.origin_href
-            puts e.href
-            puts w.response_header.status
-            failed_list << e
-            if e.http_method == :get
-              SpiderHelper.direct_http_get(e.href, e.local_path)
-            elsif e.http_method == :post
-              SpiderHelper.direct_http_post(e.href, e.local_path, e.params)
-            end
-          }
-          multi.add e.local_path, w
-        # end
+          end
+        }
+        w.errback {
+          puts "errback:#{w.response_header}"
+          puts e.origin_href
+          puts e.href
+          puts w.response_header.status
+          failed_list << e
+          if e.http_method == :get
+            SpiderHelper.direct_http_get(e.href, e.local_path)
+          elsif e.http_method == :post
+            SpiderHelper.direct_http_post(e.href, e.local_path, e.params)
+          end
+        }
+        multi.add e.local_path, w
       end
 
       cb = Proc.new do
         end_time = Time.now
         puts "use time:#{end_time-begin_time} seconds"
         if callback.nil?
-          puts "success size:#{self.succeed_size}"
-          puts "failed size:#{self.failed_size}"
-          EventMachine.stop
+          stop_machine
         else
           callback.call(multi, succeed_list, failed_list)
         end
       end
-
       link_struct_list.each &for_each_proc
       multi.callback &cb
     end
 
+    def stop_machine
+      puts "success size:#{@@succeed_size}"
+      puts "failed size:#{@@failed_size}"
+      @@end_time = Time.now
+      puts "total use time:#{@@end_time-@@begin_time} seconds"
+      EventMachine.stop
+      @@url_set.clear
+    end
+
+    def get_next_task
+      todo = []
+
+      until todo.size >= @@max || @@down_list.empty? do
+        e = @@down_list.shift
+        if @@url_set.add?(e.href)
+          todo << e
+        end
+      end
+
+      return todo
+    end
+
     def complete(multi, success_list, failed_list)
       @@succeed_size += success_list.size
       @@failed_size += failed_list.size
@@ -156,14 +165,13 @@ module ListSpider
         e.parse_method.call(e.local_path, e.extra_data) if e.parse_method
       end
 
-      todo = @@down_list.slice!(0, @@max)
+      todo = get_next_task
+
       if todo.empty?
-        puts "success size:#{@@succeed_size}"
-        puts "failed size:#{@@failed_size}"
-        EventMachine.stop
+        stop_machine
       else
         if @@inter_val != 0
-          if success_list.size != 0 || failed_list.size !=0
+          if success_list.size != 0 || failed_list.size != 0
             if @@inter_val == RANDOM_TIME
               sleep(rand(@@random_time_range))
             else
@@ -177,8 +185,13 @@ module ListSpider
 
     def event_machine_start_list(down_list, callback = nil)
       EventMachine.run {
+        @@begin_time = Time.now
         if down_list.empty?
-          callback.call(nil, [], []) if callback
+          if callback
+            callback.call(nil, [], [])
+          else
+            stop_machine
+          end
         else
           event_machine_down(down_list, callback)
         end
@@ -199,7 +212,7 @@ module ListSpider
 
     def get_list(down_list, inter_val: 0, max: 30)
       @@down_list = []
-      
+
       need_down_list = filter_list(down_list)
 
       @@down_list = @@down_list + need_down_list
@@ -210,7 +223,7 @@ module ListSpider
       @@failed_size = 0
 
       puts "total size:#{@@down_list.size}"
-      event_machine_start_list(@@down_list.slice!(0, @@max), method(:complete))
+      event_machine_start_list(get_next_task, method(:complete))
     end
 
     def get_one(task)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: list_spider
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Charles Zhang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-04 00:00:00.000000000 Z
+date: 2016-05-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: em-http-request