list_spider 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dffcea81ca28eb242e97c493175682e01ddddbc6
-  data.tar.gz: c721f41b7f763988a3b29745d303f124cea0ca71
+  metadata.gz: 38d3b1cc9c12998f7c0112f9f328d93e0c3909c8
+  data.tar.gz: 2e46c3201a979f17f3bafcbd7c2a215c666b29aa
 SHA512:
-  metadata.gz: 99714407feee9e92043d8be1e19dc41bfc50c76f588a067ffc953ce9bea9d28dd42581c70a88c53815529f02036ea192b371b49a6a4ea20b42257728bc3ccb8b
-  data.tar.gz: 18e625bd37040159fc1394bf93ab9d88464ac6459d8da51ed975a91367c2458e0f3f76c483fbee9362979c81a45c0508e29f4acae49a6896a0cfce7e5de1779c
+  metadata.gz: 25a9be2402259455a04e5c1cee6767de829841670a0967a2435e7a9616e94552e71371ab8ef69fd5e57fd640f71b205c2ae894798682a3583ec80ad2d7ac2b6d
+  data.tar.gz: cbe88b9d30421f6c1cf600d54755ad7df699e2186003c8df76b748317a3bedadd1b8c3536434c6cc11e34a9a59ece4ed4cfbfe712238538ab595d3ac4a121654

data/lib/file_filter.rb

@@ -0,0 +1,71 @@
+
+class FileFilter
+  # 4033
+  # 920
+  def initialize(dir_pattern, size_threshold: 1000, cust_judge: nil, process_block: nil)
+    @dir_pattern = dir_pattern
+    @size_threshold = size_threshold
+    if cust_judge
+      @cust_judge = cust_judge
+    else
+      @cust_judge = method(:default_judge)
+    end
+    @total = 0
+    @process_block = process_block
+  end
+
+  def default_judge(f)
+    File.size(f) <= @size_threshold
+  end
+
+  def filter_file(f)
+    if @cust_judge.call(f)
+      @total += 1
+      @process_block.call(f)
+    end
+  end
+
+  def start
+    Dir.glob(@dir_pattern) do |f|
+      filter_file(f)
+    end
+    puts "total:#{@total}"
+  end
+
+  def self.delete(dir_pattern, size_threshold: 1000, cust_judge: nil)
+    FileFilter.new(dir_pattern,
+                   size_threshold: size_threshold,
+                   cust_judge: cust_judge,
+                   process_block:
+                   proc do |f|
+                     puts "deleted file: #{f}"
+                     File.delete(f)
+                   end
+                   ).start
+  end
+
+  def self.check(dir_pattern, size_threshold: 1000, cust_judge: nil)
+    FileFilter.new(dir_pattern,
+                   size_threshold: size_threshold,
+                   cust_judge: cust_judge,
+                   process_block:
+                   proc do |f|
+                     puts "filterd file: #{f}"
+                   end
+                   ).start
+  end
+
+  def self.check_save_result(dir_pattern, save_file_name: 'filtered_file.txt', size_threshold: 1000, cust_judge: nil)
+    result_file = File.open(save_file_name, 'wt')
+    FileFilter.new(dir_pattern,
+                   size_threshold: size_threshold,
+                   cust_judge: cust_judge,
+                   process_block:
+                   proc do |f|
+                     puts "filterd file: #{f}"
+                     result_file << f << "\n"
+                   end
+                   ).start
+  end
+
+end

data/lib/list_spider.rb

@@ -4,10 +4,10 @@ require 'fileutils'
 require 'set'
 require "addressable/uri"
 require File.expand_path('../spider_helper', __FILE__)
-require File.expand_path('../delete_unvalid', __FILE__)
+require File.expand_path('../file_filter', __FILE__)
 
 class TaskStruct
-  def initialize(href, local_path, http_method: :get, params: {}, extra_data: nil, parse_method: nil)
+  def initialize(href, local_path, http_method: :get, params: {}, extra_data: nil, parse_method: nil, header: nil)
     @origin_href = href
     @href = href
     if @href.class == "".class
@@ -18,13 +18,14 @@ class TaskStruct
     @params = params
     @extra_data = extra_data
     @parse_method = parse_method
+    @header = header
   end
 
   def == (o)
-    o.class == self.class && o.href == href && o.local_path == local_path && o.http_method == http_method && o.params == params && o.extra_data == extra_data
+    o.class == self.class && o.href == href && o.local_path == local_path && o.http_method == http_method && o.params == params && o.extra_data == extra_data && o.header == header
   end
 
-  attr_accessor :origin_href , :href, :local_path, :http_method, :params, :extra_data, :parse_method, :request_object
+  attr_accessor :origin_href , :href, :local_path, :http_method, :params, :extra_data, :parse_method, :request_object, :header
 
 end
 
@@ -73,7 +74,12 @@ module ListSpider
       for_each_proc = proc do |e|
         opt = {}
         opt = {:redirects => @max_redirects}
-        opt[:head] = @header_option if defined? @header_option
+        if e.header
+          opt[:head] = e.header
+        elsif defined? @header_option
+          opt[:head] = @header_option 
+        end
+
         if e.http_method == :post
           opt[:body] = e.params unless e.params.empty?
           if @connection_opts
@@ -182,7 +188,7 @@ module ListSpider
 
           pm.call(e.local_path, e.extra_data, res_header, req)
         else
-          puts "Error! The number of arguments is:#{pm.arity}. While expected number is 1, 2, 3"
+          puts "Error! The number of arguments is:#{pm.arity}. While expected number is 1, 2, 3, 4"
         end
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: list_spider
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Charles Zhang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-13 00:00:00.000000000 Z
+date: 2016-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: em-http-request
@@ -56,7 +56,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/delete_unvalid.rb
+- lib/file_filter.rb
 - lib/list_spider.rb
 - lib/spider_helper.rb
 homepage: https://github.com/chinazhangchao/list_spider

data/lib/delete_unvalid.rb

@@ -1,40 +0,0 @@
-
-class DeleteUnvalid
-# 4033
-# 920
-  def initialize(dir_pattern, size_threshold: 1000, cust_judge: nil)
-    @dir_pattern = dir_pattern
-    @size_threshold = size_threshold
-    if cust_judge
-      @cust_judge = cust_judge
-    else
-      @cust_judge = method(:default_judge)
-    end
-    @total = 0
-  end
-  
-  def default_judge(f)
-    File.size(f) <= @size_threshold
-  end
-
-  def delete_unvaild(f)
-    if @cust_judge.call(f)
-      @total += 1
-      puts "deleted file: #{f}"
-      File.delete(f)
-    end
-  end
-
-  def start
-    Dir.glob(@dir_pattern) do |f|
-      # puts f
-      delete_unvaild(f)
-    end
-    puts "delete total:#{@total}"
-  end
-
-  def self.delete(dir_pattern, size_threshold: 1000, cust_judge: nil)
-    DeleteUnvalid.new(dir_pattern, size_threshold: size_threshold, cust_judge: cust_judge).start
-  end
-
-end