list_spider 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0f897ec31adc2d2a6c9713729030e8decc85cb9d
-  data.tar.gz: c81ed9cab5fc1bbd4395aeb9692924887042c27f
+  metadata.gz: f73a2e9b358cac55336907ac76ebdb666b9d31f5
+  data.tar.gz: 2df50eff29a1963224ca3f7d0cd9b3ac0c89156f
 SHA512:
-  metadata.gz: 907529b23336256e2c72232ec2b413c50ebee99df49695bac368ae4dc029afaa81424fe542c63d6c7247714964382e2eaeaae0ea34ab51658e34ce67e5c7b9e7
-  data.tar.gz: 22dff7b012b1c12f8cefb50606e0a9b0874c5b013a305c268ae2a950374760ba77ed24af075f6ca5217e128d905b31bae754107e0cf063e0d90171dd51b94f20
+  metadata.gz: 2cb02f9eb8593a05cc6b0a0c9d015ad93bf08663750d3dfe3007c30febfaa47d57c222960eba3b8e9275fd1f5acb942278180c8b613d8ffc0d983333f059ea8a
+  data.tar.gz: a1800a9b27c769adbae11bc8e3f08e5d57d15b3b345d19acbd329142048e742acab8878328e3c8a9053a957c3c27ecf14cd067848d1bc056f552386243c33730

data/lib/list_spider.rb

@@ -74,4 +74,8 @@ class ListSpider
     ListSpider.new(down_list, inter_val: inter_val, max: max).start
   end
 
+  def self.get_one(task)
+    ListSpider.new([task]).start
+  end
+
 end

data/lib/spider_base.rb

@@ -7,6 +7,7 @@ require "addressable/uri"
 
 class TaskStruct
   def initialize(href, local_path, http_method: :get, params: {}, extra_data: nil, parse_method: nil)
+    @origin_href = href
     @href = href
     if @href.class == "".class
       @href = SpiderHelper.string_to_uri(@href)
@@ -22,7 +23,7 @@ class TaskStruct
     o.class == self.class && o.href == href && o.local_path == local_path && o.http_method == http_method && o.params == params && o.extra_data == extra_data
   end
 
-  attr_accessor :href, :local_path, :http_method, :params, :extra_data, :parse_method
+  attr_accessor :origin_href , :href, :local_path, :http_method, :params, :extra_data, :parse_method
 
 end
 
@@ -112,6 +113,7 @@ module SpiderBase
           }
           w.errback {
             puts "errback:#{w.response_header}"
+            puts e.origin_href
             puts e.href
             puts w.response_header.status
             failed_list << e

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: list_spider
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Charles Zhang