list_spider 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ffb9c26b0fc7db4fd6a2a3b831832964afbaa949
-  data.tar.gz: 2c829fb860ad3c0c8b3825c1575988cadfccaa15
+  metadata.gz: aa975b345606cc3995e08310737afd1338ad4c80
+  data.tar.gz: a5d15c9dcf5f1f78395f75911370617c9481b1ea
 SHA512:
-  metadata.gz: 1673f2f5daed52295ee567c2d1c6b672f1bdf532fd31a334a704cd9ac5fbdcef9aea0f27e9828cd816769f74bfcbd8498294f1d190e73defb0fb6c15fb990e04
-  data.tar.gz: 31247ae66dfac70a69671c336dcc942812d26fd9572c2e527dbd511bf40ed680304fcb24d7f1fac01e51b52d666061ffacaaae49c960a51aecff61f07ddcf9be
+  metadata.gz: 8fd86f3186bf640d2d895028d1373984046183e941619bf01b0c2d611deb7a62d4a52098da23e3299d20949212b776a6b368a4923c37e152d2e3e3038b433a1a
+  data.tar.gz: 722ee573500abe11549e679b3def10def9830ef58ae6f4bf24e0e98d0fc872524d97c964ce17df3dc9718eedfb3e66067f09fdeda89bce6d39f0c2fa1bbca750

data/lib/list_spider.rb

@@ -24,7 +24,7 @@ class TaskStruct
     o.class == self.class && o.href == href && o.local_path == local_path && o.http_method == http_method && o.params == params && o.extra_data == extra_data
   end
 
-  attr_accessor :origin_href , :href, :local_path, :http_method, :params, :extra_data, :parse_method
+  attr_accessor :origin_href , :href, :local_path, :http_method, :params, :extra_data, :parse_method, :request_object
 
 end
 
@@ -37,10 +37,10 @@ module ListSpider
 
   @random_time_range = 3..10
   @conver_to_utf8 = false
-  @connection_opts = {connect_timeout: 2*60}
+  @connection_opts = {connect_timeout: 60}
   @overwrite_exist = false
   @max_redirects = 10
-  @@url_set = Set.new
+  @@local_path_set = Set.new
 
   class << self
 
@@ -61,7 +61,7 @@ module ListSpider
     end
 
     def set_header_option(header_option)
-      @@header_option = optHash
+      @@header_option = header_option
     end
 
     def event_machine_down(link_struct_list, callback = nil)
@@ -90,6 +90,8 @@ module ListSpider
           end
         end
 
+        e.request_object = w
+
         w.callback {
           s = w.response_header.status
           puts s
@@ -122,7 +124,15 @@ module ListSpider
             SpiderHelper.direct_http_post(e.href, e.local_path, e.params)
           end
         }
-        multi.add e.local_path, w
+
+        begin
+          multi.add e.local_path, w
+        rescue Exception => exception
+          puts exception
+          puts e.href
+          puts e.local_path
+          stop_machine
+        end
       end
 
       cb = Proc.new do
@@ -144,27 +154,36 @@ module ListSpider
       @@end_time = Time.now
       puts "total use time:#{@@end_time-@@begin_time} seconds"
       EventMachine.stop
-      @@url_set.clear
+      @@local_path_set.clear
     end
 
     def get_next_task
-      todo = []
+      return @@down_list.shift(@@max)
+    end
 
-      until todo.size >= @@max || @@down_list.empty? do
-        e = @@down_list.shift
-        if @@url_set.add?(e.href)
-          todo << e
+    def call_parse_method(e)
+      pm = e.parse_method
+      if pm
+        case pm.arity
+        when 1
+          pm.call(e.local_path)
+        when 2
+          pm.call(e.local_path, e.extra_data)
+        when 3
+          res_header = nil
+          res_header = e.request_object.response_header if e.request_object
+          pm.call(e.local_path, e.extra_data, res_header)
+        else
+          puts "Error! The number of arguments is:#{pm.arity}. While expected number is 1, 2, 3"
         end
       end
-
-      return todo
     end
 
     def complete(multi, success_list, failed_list)
       @@succeed_size += success_list.size
       @@failed_size += failed_list.size
       success_list.each do |e|
-        e.parse_method.call(e.local_path, e.extra_data) if e.parse_method
+        call_parse_method(e)
       end
 
       todo = get_next_task
@@ -204,8 +223,8 @@ module ListSpider
       need_down_list = []
       down_list.each do |ts|
         if !@overwrite_exist && File.exist?(ts.local_path)
-          ts.parse_method.call(ts.local_path, ts.extra_data) if ts.parse_method
-        else
+          call_parse_method(ts)
+        elsif @@local_path_set.add?(ts.local_path)
           need_down_list << ts
         end
       end
@@ -243,6 +262,11 @@ module ListSpider
         puts "error task type:#{task.class}"
       end
     end
+  end
 
+  Signal.trap("INT") do
+    ListSpider.stop_machine
+    exit!
   end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: list_spider
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - Charles Zhang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-08 00:00:00.000000000 Z
+date: 2016-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: em-http-request