list_spider 0.1.9 → 0.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4b43422e1a52997518a53a38c021f68700ec6d3f
-  data.tar.gz: cefa5535e5870a221b04a47d6b1f6ec177392caa
+  metadata.gz: 540f6b583b38b40fe7c6096d372a43b93dd20002
+  data.tar.gz: 99b187103019165b5c87dd094c748d0b889d306b
 SHA512:
-  metadata.gz: 2c561ce1828e592a538e1747dccc1da39c1946f182ffc19212fd8284b90b0c1f43e7e584ff3bb1f416b984bc04328901d887bf4fd294f20448ad98a7d7b0c1dd
-  data.tar.gz: 55d6a8f01bb1a325d365d42f33123289881f37e985768ef75538a6f1a1f044b248cf42f5685f6974591f86f0ba140e78a693cac4af4981c6b564cd3a131568ef
+  metadata.gz: 6bd249021510b182d67912573feb5f9204c82f201340d7515f6dcb0d024fd9920abd1acb7724c6670967455a1a0b92426f40cf3d7550da40f00decfdd8e7f903
+  data.tar.gz: 6a28ca654fbb5f906dacfcccc121db52c09b6efa70c1e3724a1db3937e7be15922286c476ff390c48c227a7b492866c39ef1c3187e9636a0fd3ec9ac391d88fc

data/lib/list_spider.rb

@@ -40,11 +40,11 @@ module ListSpider
   @connection_opts = {connect_timeout: 60}
   @overwrite_exist = false
   @max_redirects = 10
-  @@local_path_set = Set.new
+  @local_path_set = Set.new
 
   class << self
 
-    attr_accessor :random_time_range, :conver_to_utf8, :overwrite_exist, :max_redirects
+    attr_accessor :conver_to_utf8, :overwrite_exist, :max_redirects
 
     def set_proxy(proxy_addr, proxy_port, username: nil, password: nil)
       @connection_opts = {
@@ -61,7 +61,7 @@ module ListSpider
     end
 
     def set_header_option(header_option)
-      @@header_option = header_option
+      @header_option = header_option
     end
 
     def event_machine_down(link_struct_list, callback = nil)
@@ -73,7 +73,7 @@ module ListSpider
       for_each_proc = proc do |e|
         opt = {}
         opt = {:redirects => @max_redirects}
-        opt[:head] = @@header_option if defined? @@header_option
+        opt[:head] = @header_option if defined? @header_option
         if e.http_method == :post
           opt[:body] = e.params unless e.params.empty?
           if @connection_opts
@@ -149,16 +149,16 @@ module ListSpider
     end
 
     def stop_machine
-      puts "success size:#{@@succeed_size}"
-      puts "failed size:#{@@failed_size}"
-      @@end_time = Time.now
-      puts "total use time:#{@@end_time-@@begin_time} seconds"
+      puts "success size:#{@succeed_size}"
+      puts "failed size:#{@failed_size}"
+      @end_time = Time.now
+      puts "total use time:#{@end_time-@begin_time} seconds"
       EventMachine.stop
-      @@local_path_set.clear
+      @local_path_set.clear
     end
 
     def get_next_task
-      return @@down_list.shift(@@max)
+      return @down_list.shift(@max)
     end
 
     def call_parse_method(e)
@@ -188,8 +188,8 @@ module ListSpider
     end
 
     def complete(multi, success_list, failed_list)
-      @@succeed_size += success_list.size
-      @@failed_size += failed_list.size
+      @succeed_size += success_list.size
+      @failed_size += failed_list.size
       success_list.each do |e|
         call_parse_method(e)
       end
@@ -199,12 +199,12 @@ module ListSpider
       if todo.empty?
         stop_machine
       else
-        if @@interval != 0
+        if @interval != 0
           if success_list.size != 0 || failed_list.size != 0
-            if @@interval == RANDOM_TIME
+            if @interval == RANDOM_TIME
               sleep(rand(@random_time_range))
             else
-              sleep(@@interval)
+              sleep(@interval)
             end
           end
         end
@@ -214,7 +214,7 @@ module ListSpider
 
     def event_machine_start_list(down_list, callback = nil)
       EventMachine.run {
-        @@begin_time = Time.now
+        @begin_time = Time.now
         if down_list.empty?
           if callback
             callback.call(nil, [], [])
@@ -232,7 +232,7 @@ module ListSpider
       down_list.each do |ts|
         if !@overwrite_exist && File.exist?(ts.local_path)
           call_parse_method(ts)
-        elsif @@local_path_set.add?(ts.local_path)
+        elsif @local_path_set.add?(ts.local_path)
           need_down_list << ts
         end
       end
@@ -245,18 +245,18 @@ module ListSpider
         interval = RANDOM_TIME
       end
 
-      @@down_list = []
+      @down_list = []
 
       need_down_list = filter_list(down_list)
 
-      @@down_list = @@down_list + need_down_list
-      @@interval = interval
-      @@max = max
-      @@max = @@down_list.size if @@max == NO_LIMIT_CONCURRENT
-      @@succeed_size = 0
-      @@failed_size = 0
+      @down_list = @down_list + need_down_list
+      @interval = interval
+      @max = max
+      @max = @down_list.size if @max == NO_LIMIT_CONCURRENT
+      @succeed_size = 0
+      @failed_size = 0
 
-      puts "total size:#{@@down_list.size}"
+      puts "total size:#{@down_list.size}"
       event_machine_start_list(get_next_task, method(:complete))
     end
 
@@ -267,10 +267,10 @@ module ListSpider
     def add_task(task)
       if task.is_a?Array
         need_down_list = filter_list(task)
-        @@down_list = @@down_list + need_down_list
+        @down_list = @down_list + need_down_list
       elsif task.is_a?TaskStruct
         need_down_list = filter_list([task])
-        @@down_list = @@down_list + need_down_list
+        @down_list = @down_list + need_down_list
       else
         puts "error task type:#{task.class}"
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: list_spider
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.10
 platform: ruby
 authors:
 - Charles Zhang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-12 00:00:00.000000000 Z
+date: 2016-05-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: em-http-request