list_spider 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2daa1e0b7fbebdfcc8d4b7299d7c2378953721e9
-  data.tar.gz: c7975f3e4eda6c80204ce92713b39c6eca0e2c7e
+  metadata.gz: 9e16f58e39ec18d181da107dfdc76626c23dc44c
+  data.tar.gz: c6fd2518111983f1bdcf9c471a93cfe279a722fe
 SHA512:
-  metadata.gz: 87bca1533b16e09032b02dbb90830c1d9db03699d8fe2138574e389df63c2293dc8af6221f5a315d13ccd1bc09be4c10c8040ee13145d70ee523df37b6720912
-  data.tar.gz: fd29efa797377f01b99d7d878bec5b9263628b21f36090e08b9e62aa4196ad2273c704423b0dc15afb4e853f5757c58aa3f64a689d51c32fffe222aa6bb3279a
+  metadata.gz: 8d3374e67436fc68afdd770c9ece27cfd734e82dc74917e7db226f311f743cbd51c69129f0c400bad6ec602c15f1cb08697c2b85a91805c8c3c549e519a68f74
+  data.tar.gz: 9ef2fdb5d99fcf018ce02eeb395eedf4e72917018b21b48a23ae12601fa7b2b566ba6c08595674900326d8f4d1d18c6e992c2cf544f468dffa96f3cbeefecf7b

data/lib/file_filter.rb

@@ -5,11 +5,7 @@ class FileFilter
   def initialize(dir_pattern, size_threshold: 1000, cust_judge: nil, process_block: nil)
     @dir_pattern = dir_pattern
     @size_threshold = size_threshold
-    if cust_judge
-      @cust_judge = cust_judge
-    else
-      @cust_judge = method(:default_judge)
-    end
+    @cust_judge = cust_judge ? cust_judge : method(:default_judge)
     @total = 0
     @process_block = process_block
   end
@@ -33,39 +29,42 @@ class FileFilter
   end
 
   def self.delete(dir_pattern, size_threshold: 1000, cust_judge: nil)
-    FileFilter.new(dir_pattern,
-                   size_threshold: size_threshold,
-                   cust_judge: cust_judge,
-                   process_block:
-                   proc do |f|
-                     puts "deleted file: #{f}"
-                     File.delete(f)
-                   end
-                   ).start
+    FileFilter.new(
+      dir_pattern,
+      size_threshold: size_threshold,
+      cust_judge: cust_judge,
+      process_block:
+      proc do |f|
+        puts "deleted file: #{f}"
+        File.delete(f)
+      end
+    ).start
   end
 
   def self.check(dir_pattern, size_threshold: 1000, cust_judge: nil)
-    FileFilter.new(dir_pattern,
-                   size_threshold: size_threshold,
-                   cust_judge: cust_judge,
-                   process_block:
-                   proc do |f|
-                     puts "filterd file: #{f}"
-                   end
-                   ).start
+    FileFilter.new(
+      dir_pattern,
+      size_threshold: size_threshold,
+      cust_judge: cust_judge,
+      process_block:
+      proc do |f|
+        puts "filterd file: #{f}"
+      end
+    ).start
   end
 
   def self.check_save_result(dir_pattern, save_file_name: 'filtered_file.txt', size_threshold: 1000, cust_judge: nil)
     result_file = File.open(save_file_name, 'wt')
-    FileFilter.new(dir_pattern,
-                   size_threshold: size_threshold,
-                   cust_judge: cust_judge,
-                   process_block:
-                   proc do |f|
-                     puts "filterd file: #{f}"
-                     result_file << f << "\n"
-                   end
-                   ).start
+    FileFilter.new(
+      dir_pattern,
+      size_threshold: size_threshold,
+      cust_judge: cust_judge,
+      process_block:
+      proc do |f|
+        puts "filterd file: #{f}"
+        result_file << f << "\n"
+      end
+    ).start
+    result_file.close
   end
-
 end

data/lib/list_spider.rb

@@ -2,7 +2,7 @@ require 'em-http-request'
 require 'nokogiri'
 require 'fileutils'
 require 'set'
-require "addressable/uri"
+require 'addressable/uri'
 require File.expand_path('../spider_helper', __FILE__)
 require File.expand_path('../file_filter', __FILE__)
 
@@ -10,9 +10,7 @@ class TaskStruct
   def initialize(href, local_path, http_method: :get, params: {}, extra_data: nil, parse_method: nil, header: nil)
     @origin_href = href
     @href = href
-    if @href.class == "".class
-      @href = SpiderHelper.string_to_uri(@href)
-    end
+    @href = SpiderHelper.string_to_uri(@href) if @href.class == ''.class
     @local_path = local_path
     @http_method = http_method
     @params = params
@@ -21,16 +19,14 @@ class TaskStruct
     @header = header
   end
 
-  def == (o)
-    o.class == self.class && o.href == href && o.local_path == local_path && o.http_method == http_method && o.params == params && o.extra_data == extra_data && o.header == header
+  def ==(other)
+    other.class == self.class && other.href == href && other.local_path == local_path && other.http_method == http_method && other.params == params && other.extra_data == extra_data && other.header == header
   end
 
-  attr_accessor :origin_href , :href, :local_path, :http_method, :params, :extra_data, :parse_method, :request_object, :header
-
+  attr_accessor :origin_href, :href, :local_path, :http_method, :params, :extra_data, :parse_method, :request_object, :header
 end
 
 module ListSpider
-
   RANDOM_TIME = -1
   NO_LIMIT_CONCURRENT = -1
   DEFAULT_CONCURRNET_MAX = 50
@@ -38,21 +34,20 @@ module ListSpider
 
   @random_time_range = 3..10
   @conver_to_utf8 = false
-  @connection_opts = {connect_timeout: 60}
+  @connection_opts = { connect_timeout: 60 }
   @overwrite_exist = false
   @max_redirects = 10
   @local_path_set = Set.new
 
   class << self
-
     attr_accessor :conver_to_utf8, :overwrite_exist, :max_redirects
 
     def set_proxy(proxy_addr, proxy_port, username: nil, password: nil)
       @connection_opts = {
-        :proxy => {
-        :host => proxy_addr,
-        :port => proxy_port
-      }
+        proxy: {
+          host: proxy_addr,
+          port: proxy_port
+        }
       }
       @connection_opts[:proxy][:authorization] = [username, password] if username && password
     end
@@ -71,100 +66,102 @@ module ListSpider
       multi = EventMachine::MultiRequest.new
       begin_time = Time.now
 
-      for_each_proc = proc do |e|
-        opt = {}
-        opt = {:redirects => @max_redirects}
-        if e.header
-          opt[:head] = e.header
-        elsif defined? @header_option
-          opt[:head] = @header_option 
-        end
-
-        if e.http_method == :post
-          opt[:body] = e.params unless e.params.empty?
-          if @connection_opts
-            w = EventMachine::HttpRequest.new(e.href, @connection_opts).post opt
-          else
-            w = EventMachine::HttpRequest.new(e.href).post opt
+      for_each_proc =
+        proc do |e|
+          opt = { redirects: @max_redirects }
+          if e.header
+            opt[:head] = e.header
+          elsif defined? @header_option
+            opt[:head] = @header_option
           end
-        else
-          if @connection_opts
-            opt[:query] = e.params unless e.params.empty?
-            w = EventMachine::HttpRequest.new(e.href, @connection_opts).get opt
+
+          if e.http_method == :post
+            opt[:body] = e.params unless e.params.empty?
+            w =
+              if @connection_opts
+                EventMachine::HttpRequest.new(e.href, @connection_opts).post opt
+              else
+                EventMachine::HttpRequest.new(e.href).post opt
+              end
           else
-            w = EventMachine::HttpRequest.new(e.href).get opt
+            if @connection_opts
+              opt[:query] = e.params unless e.params.empty?
+              w = EventMachine::HttpRequest.new(e.href, @connection_opts).get opt
+            else
+              w = EventMachine::HttpRequest.new(e.href).get opt
+            end
           end
-        end
 
-        e.request_object = w
-
-        w.callback {
-          s = w.response_header.status
-          puts s
-          if s != 404
-            local_dir = File.dirname(e.local_path)
-            FileUtils.mkdir_p(local_dir) unless Dir.exist?(local_dir)
-            begin
-              File.open(e.local_path, "w") do |f|
-                if @conver_to_utf8 == true
-                  f << SpiderHelper.to_utf8( w.response)
-                else
-                  f << w.response
+          e.request_object = w
+
+          w.callback do
+            s = w.response_header.status
+            puts s
+            if s != 404
+              local_dir = File.dirname(e.local_path)
+              FileUtils.mkdir_p(local_dir) unless Dir.exist?(local_dir)
+              begin
+                File.open(e.local_path, 'w') do |f|
+                  f << if @conver_to_utf8 == true
+                         SpiderHelper.to_utf8(w.response)
+                       else
+                         w.response
+                       end
                 end
+                succeed_list << e
+              rescue => e
+                puts e
               end
-              succeed_list << e
-            rescue Exception => e
-              puts e
             end
           end
-        }
-        w.errback {
-          puts "errback:#{w.response_header}"
-          puts e.origin_href
-          puts e.href
-          puts w.response_header.status
-          failed_list << e
-          if e.http_method == :get
-            SpiderHelper.direct_http_get(e.href, e.local_path)
-          elsif e.http_method == :post
-            SpiderHelper.direct_http_post(e.href, e.local_path, e.params)
+          w.errback do
+            puts "errback:#{w.response_header}"
+            puts e.origin_href
+            puts e.href
+            puts w.response_header.status
+            failed_list << e
+            if e.http_method == :get
+              SpiderHelper.direct_http_get(e.href, e.local_path)
+            elsif e.http_method == :post
+              SpiderHelper.direct_http_post(e.href, e.local_path, e.params)
+            end
           end
-        }
 
-        begin
-          multi.add e.local_path, w
-        rescue Exception => exception
-          puts exception
-          puts e.href
-          puts e.local_path
-          stop_machine
+          begin
+            multi.add e.local_path, w
+          rescue => exception
+            puts exception
+            puts e.href
+            puts e.local_path
+            stop_machine
+          end
         end
-      end
 
-      cb = Proc.new do
-        end_time = Time.now
-        puts "use time:#{end_time-begin_time} seconds"
-        if callback.nil?
-          stop_machine
-        else
-          callback.call(multi, succeed_list, failed_list)
+      cb =
+        proc do
+          end_time = Time.now
+          puts "use time:#{end_time - begin_time} seconds"
+          if callback.nil?
+            stop_machine
+          else
+            callback.call(multi, succeed_list, failed_list)
+          end
         end
-      end
-      link_struct_list.each &for_each_proc
-      multi.callback &cb
+      link_struct_list.each(&for_each_proc)
+      multi.callback(&cb)
     end
 
     def stop_machine
       puts "success size:#{@succeed_size}"
       puts "failed size:#{@failed_size}"
       @end_time = Time.now
-      puts "total use time:#{@end_time-@begin_time} seconds"
+      puts "total use time:#{@end_time - @begin_time} seconds"
       EventMachine.stop
       @local_path_set.clear
     end
 
-    def get_next_task
-      return @down_list.shift(@max)
+    def next_task
+      @down_list.shift(@max)
     end
 
     def call_parse_method(e)
@@ -193,20 +190,20 @@ module ListSpider
       end
     end
 
-    def complete(multi, success_list, failed_list)
+    def complete(_multi, success_list, failed_list)
       @succeed_size += success_list.size
       @failed_size += failed_list.size
       success_list.each do |e|
         call_parse_method(e)
       end
 
-      todo = get_next_task
+      todo = next_task
 
       if todo.empty?
         stop_machine
       else
         if @interval != 0
-          if success_list.size != 0 || failed_list.size != 0
+          if !success_list.empty? || !failed_list.empty?
             if @interval == RANDOM_TIME
               sleep(rand(@random_time_range))
             else
@@ -219,7 +216,7 @@ module ListSpider
     end
 
     def event_machine_start_list(down_list, callback = nil)
-      EventMachine.run {
+      EventMachine.run do
         @begin_time = Time.now
         if down_list.empty?
           if callback
@@ -230,7 +227,7 @@ module ListSpider
         else
           event_machine_down(down_list, callback)
         end
-      }
+      end
     end
 
     def filter_list(down_list)
@@ -242,7 +239,7 @@ module ListSpider
           need_down_list << ts
         end
       end
-      return need_down_list
+      need_down_list
     end
 
     def get_list(down_list, interval: DEFAULT_INTERVAL, max: DEFAULT_CONCURRNET_MAX)
@@ -255,7 +252,7 @@ module ListSpider
 
       need_down_list = filter_list(down_list)
 
-      @down_list = @down_list + need_down_list
+      @down_list += need_down_list
       @interval = interval
       @max = max
       @max = @down_list.size if @max == NO_LIMIT_CONCURRENT
@@ -263,7 +260,7 @@ module ListSpider
       @failed_size = 0
 
       puts "total size:#{@down_list.size}"
-      event_machine_start_list(get_next_task, method(:complete))
+      event_machine_start_list(next_task, method(:complete))
     end
 
     def get_one(task, interval: DEFAULT_INTERVAL, max: DEFAULT_CONCURRNET_MAX)
@@ -273,19 +270,18 @@ module ListSpider
     def add_task(task)
       if task.is_a?Array
         need_down_list = filter_list(task)
-        @down_list = @down_list + need_down_list
+        @down_list += need_down_list
       elsif task.is_a?TaskStruct
         need_down_list = filter_list([task])
-        @down_list = @down_list + need_down_list
+        @down_list += need_down_list
       else
         puts "error task type:#{task.class}"
       end
     end
   end
 
-  Signal.trap("INT") do
+  Signal.trap('INT') do
     ListSpider.stop_machine
     exit!
   end
-
 end

data/lib/spider_helper.rb

@@ -2,50 +2,46 @@ require 'rchardet'
 require 'net/http'
 
 module SpiderHelper
-
   class << self
-
     def direct_http_get(href, local_path, params: nil, header: nil)
-      if href.class == "".class
-        href = string_to_uri(href)
-      end
+      href = string_to_uri(href) if href.class == ''.class
 
       begin
         href.query = URI.encode_www_form(params) if params
         req = Net::HTTP::Get.new(href)
-        header.each{|k,v| req[k] = v} if header
+        header.each { |k, v| req[k] = v } if header
 
-        res = Net::HTTP.start(href.hostname, href.port) do |http|
-          http.request(req)
-        end
+        res =
+          Net::HTTP.start(href.hostname, href.port) do |http|
+            http.request(req)
+          end
 
         if res.is_a?(Net::HTTPSuccess)
           local_dir = File.dirname(local_path)
           FileUtils.mkdir_p(local_dir) unless Dir.exist?(local_dir)
           File.write(local_path, res.body)
-          puts "succeed"
+          puts 'succeed'
         else
           puts res
         end
-      rescue Exception => e
+      rescue => e
         puts e.backtrace
         puts e
       end
     end
 
     def direct_http_post(href, local_path, params, header: nil)
-      if href.class == "".class
-        href = string_to_uri(href)
-      end
+      href = string_to_uri(href) if href.class == ''.class
 
       begin
         req = Net::HTTP::Post.new(href)
         req.set_form_data(params)
-        header.each{|k,v| req[k] = v} if header
+        header.each { |k, v| req[k] = v } if header
 
-        res = Net::HTTP.start(href.hostname, href.port) do |http|
-          http.request(req)
-        end
+        res =
+          Net::HTTP.start(href.hostname, href.port) do |http|
+            http.request(req)
+          end
 
         if res.is_a?(Net::HTTPSuccess)
           local_dir = File.dirname(local_path)
@@ -54,7 +50,7 @@ module SpiderHelper
         else
           puts res
         end
-      rescue Exception => e
+      rescue => e
         puts e
       end
     end
@@ -70,34 +66,32 @@ module SpiderHelper
       l.normalize!
     end
 
-    BomHeaderMap = {"UTF-8" => "\xEF\xBB\xBF".force_encoding("UTF-8"),
-      "UTF-16BE"=>"\xFE\xFF".force_encoding("UTF-16BE"),
-      "UTF-16LE"=>"\xFF\xFE".force_encoding("UTF-16LE"),
-      "UTF-32BE"=>"\x00\x00\xFE\xFF".force_encoding("UTF-32BE"),
-      "UTF-32LE"=>"\xFF\xFE\x00\x00".force_encoding("UTF-32LE")}
+    BomHeaderMap = { 'UTF-8' => "\xEF\xBB\xBF".force_encoding('UTF-8'),
+                     'UTF-16BE' => "\xFE\xFF".force_encoding('UTF-16BE'),
+                     'UTF-16LE' => "\xFF\xFE".force_encoding('UTF-16LE'),
+                     'UTF-32BE' => "\x00\x00\xFE\xFF".force_encoding('UTF-32BE'),
+                     'UTF-32LE' => "\xFF\xFE\x00\x00".force_encoding('UTF-32LE') }.freeze
 
-    #此函数有时此判断有误，使用to_utf8函数直接转换
+    # 此函数有时此判断有误，使用to_utf8函数直接转换
     def smart_to_utf8(str)
       return str if str.encoding == Encoding::UTF_8
       to_utf8(str)
     end
 
     def to_utf8(str)
-      #解决windows下CharDet库编译为ASCII_8BIT，无法与UTF-8兼容问题
+      # 解决windows下CharDet库编译为ASCII_8BIT，无法与UTF-8兼容问题
       str.force_encoding(Encoding::ASCII_8BIT)
       cd = CharDet.detect(str)
-      if cd["confidence"] > 0.6
-        puts cd["encoding"]
-        str.force_encoding(cd["encoding"])
-        #移除BOM头
-        bomHeader = BomHeaderMap[cd["encoding"]]
-        str.sub!(bomHeader, "") if bomHeader
+      if cd['confidence'] > 0.6
+        puts cd['encoding']
+        str.force_encoding(cd['encoding'])
+        # 移除BOM头
+        bom_header = BomHeaderMap[cd['encoding']]
+        str.sub!(bom_header, '') if bom_header
       end
-      str.encode!(Encoding::UTF_8, :undef => :replace, :replace => "?", :invalid => :replace)
-      
-      return str
-    end
+      str.encode!(Encoding::UTF_8, undef: :replace, replace: '?', invalid: :replace)
 
+      str
+    end
   end
-
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: list_spider
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Charles Zhang