list_spider 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: facd4c45a64f0cca934e70de9cfda4f05cf1a1d9
-  data.tar.gz: b0e5940faa51be59c13a0efcfc114b43981082b7
+  metadata.gz: 624a5d306b193b1ece24867cc1ff8b8031931d90
+  data.tar.gz: 00b861e2d021dfd668275cf33a87b275fafa0def
 SHA512:
-  metadata.gz: a21daf8996b6aa27714a6407511a094f6381f739a65066077ca89bb0f4783ba0e6576b48080dbdaacee73d1a2ea4e6c77195ba832ee8f0bc28bde295f8e50b8c
-  data.tar.gz: e5bb44f131f82f480d9b5ebd146587c6473f6d3a7433fea32dd46bb74ac2c76adebc362048ee1c6228ca563ae4aa3d1b69bcc6d8735a4ee683e5c848f60944a4
+  metadata.gz: 1e739921dc0b12d47a267cb5ed01ff626055d4e5ff5589c9ec9330906ab4db051079631a23671772af51a27d5dca9c03652b8f163e0028e4ea04f4dd382c6f29
+  data.tar.gz: dd6badf6fae1780525c5c4be335021624b3ecc943d7db12c60e9dd4c4bb21cfb731c30b24148c436858eb3617c1fd997c27ad01fea59e7dc5743e1b1dcdc94d2

data/lib/list_spider.rb

@@ -33,11 +33,11 @@ module ListSpider
   RANDOM_TIME = -1
   NO_LIMIT_CONCURRENT = -1
 
-  @@random_time_range = 3..10
-  @@conver_to_utf8 = false
-  @@connection_opts = {:connect_timeout => 2*60}
-  @@overwrite_exist = false
-  @@max_redirects = 10
+  @random_time_range = 3..10
+  @conver_to_utf8 = false
+  @connection_opts = {connect_timeout: 2*60}
+  @overwrite_exist = false
+  @max_redirects = 10
   @@url_set = Set.new
 
   class << self
@@ -45,17 +45,17 @@ module ListSpider
     attr_accessor :random_time_range, :conver_to_utf8, :overwrite_exist, :max_redirects
 
     def set_proxy(proxy_addr, proxy_port, username: nil, password: nil)
-      @@connection_opts = {
+      @connection_opts = {
         :proxy => {
         :host => proxy_addr,
         :port => proxy_port
       }
       }
-      @@connection_opts[:proxy][:authorization] = [username, password] if username && password
+      @connection_opts[:proxy][:authorization] = [username, password] if username && password
     end
 
     def connect_timeout(max_connect_time)
-      @@connection_opts[:connect_timeout] = max_connect_time
+      @connection_opts[:connect_timeout] = max_connect_time
     end
 
     def set_header_option(header_option)
@@ -70,19 +70,19 @@ module ListSpider
 
       for_each_proc = proc do |e|
         opt = {}
-        opt = {:redirects => @@max_redirects}
+        opt = {:redirects => @max_redirects}
         opt[:head] = @@header_option if defined? @@header_option
         if e.http_method == :post
           opt[:body] = e.params unless e.params.empty?
-          if @@connection_opts
-            w = EventMachine::HttpRequest.new(e.href, @@connection_opts).post opt
+          if @connection_opts
+            w = EventMachine::HttpRequest.new(e.href, @connection_opts).post opt
           else
             w = EventMachine::HttpRequest.new(e.href).post opt
           end
         else
-          if @@connection_opts
+          if @connection_opts
             opt[:query] = e.params unless e.params.empty?
-            w = EventMachine::HttpRequest.new(e.href, @@connection_opts).get opt
+            w = EventMachine::HttpRequest.new(e.href, @connection_opts).get opt
           else
             w = EventMachine::HttpRequest.new(e.href).get opt
           end
@@ -96,7 +96,7 @@ module ListSpider
             FileUtils.mkdir_p(local_dir) unless Dir.exist?(local_dir)
             begin
               File.open(e.local_path, "w") do |f|
-                if @@conver_to_utf8 == true
+                if @conver_to_utf8 == true
                   f << SpiderHelper.to_utf8( w.response)
                 else
                   f << w.response
@@ -173,7 +173,7 @@ module ListSpider
         if @@inter_val != 0
           if success_list.size != 0 || failed_list.size != 0
             if @@inter_val == RANDOM_TIME
-              sleep(rand(@@random_time_range))
+              sleep(rand(@random_time_range))
             else
               sleep(@@inter_val)
             end
@@ -201,7 +201,7 @@ module ListSpider
     def filter_list(down_list)
       need_down_list = []
       down_list.each do |ts|
-        if !@@overwrite_exist && File.exist?(ts.local_path)
+        if !@overwrite_exist && File.exist?(ts.local_path)
           ts.parse_method.call(ts.local_path, ts.extra_data) if ts.parse_method
         else
           need_down_list << ts

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: list_spider
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Charles Zhang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-05 00:00:00.000000000 Z
+date: 2016-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: em-http-request