list_spider 2.0.0 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 837d9e4cb2b3aa829466cf9eaa4f48a24b5d4ff5067bbc27fb67fbdb37eec291
-  data.tar.gz: 8d378b9e3240b8d9c3bdc9c7e32aceb39a16fc63310224dc7ce6a68a2c570893
+  metadata.gz: 9ef1d9cd5edd321718137a9b9055b6e223fb870f1209271df4142930c35dd6f5
+  data.tar.gz: 983055db2e6714337f4f76ec3d279a83f46d48e689be0334c9a944bce197599c
 SHA512:
-  metadata.gz: dd2c77aa71d8ff3d7ecba93fc6e30ec158b479dcffed9e3cc744944e2bcea3cb5425fc59f85acc22573bcbe3d1eb9a0967e7d0b1e11d3c9cb8d04a58450a0a7e
-  data.tar.gz: ec0e3ac5b2a09a3986eea20c69efc31c9536d1d96f77507e50755bfa07531c4bf7303317bc657a573dd8347bd304d8e93c9adbabf868918f0bdbe56c480e82e6
+  metadata.gz: dcccc6c7761931c93bd729e58a844e127d9ac7a63075f8163c92083f7749df2e978cc745071a2c319bcc48825d2dd53a2e043e97a02329884168cc50d21f7975
+  data.tar.gz: 33cd0806b64df7e88296a9ff8ce0a3b233f6702df9cdb4c87938c7cc0b64cde2d096970903a3242c228c308d7fc2d8a35f4eff85d2262e5034de83bf7d009e85

data/Gemfile.lock

@@ -0,0 +1,41 @@
+PATH
+  remote: .
+  specs:
+    list_spider (2.0.0)
+      em-http-request (~> 1.1, >= 1.1.3)
+      nokogiri (~> 1.6, >= 1.6.7)
+      rchardet (~> 1.6, >= 1.6.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.5.2)
+      public_suffix (>= 2.0.2, < 4.0)
+    cookiejar (0.3.3)
+    em-http-request (1.1.5)
+      addressable (>= 2.3.4)
+      cookiejar (!= 0.3.1)
+      em-socksify (>= 0.3)
+      eventmachine (>= 1.0.3)
+      http_parser.rb (>= 0.6.0)
+    em-socksify (0.3.2)
+      eventmachine (>= 1.0.0.beta.4)
+    eventmachine (1.2.5)
+    http_parser.rb (0.6.0)
+    mini_portile2 (2.3.0)
+    nokogiri (1.8.2)
+      mini_portile2 (~> 2.3.0)
+    public_suffix (3.0.2)
+    rake (10.5.0)
+    rchardet (1.7.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.16)
+  list_spider!
+  rake (~> 10.0)
+
+BUNDLED WITH
+   1.16.1

data/lib/list_spider.rb

@@ -14,7 +14,7 @@ class TaskStruct
                  http_method: :get,
                  custom_data: nil, # 自定义数据
                  parse_method: nil, # 解析保存文件的回调，参数是TaskStruct对象本身
-                 # 请求成功后的回调，此时可能没有保存文件，比如301，
+                 # 请求成功后的回调，此时可能没有保存文件，比如301，404
                  # 参数是TaskStruct对象本身和对应的EventMachine::HttpRequest对象
                  # http.response_header.status 状态码
                  # http.response_header  返回头
@@ -28,9 +28,9 @@ class TaskStruct
                  overwrite_exist: false, # 是否覆盖现有文件
                  # request options
                  redirects: 3, # 重定向次数
-                 #  keepalive: nil, # （暂不支持）
+                 keepalive: nil, # （暂不支持复用）
                  file: nil, # 要上传的文件路径
-                 #  path: nil, # 请求路径，在流水线方式请求时有用（暂不支持）
+                 path: nil, # 请求路径，在流水线方式请求时有用（暂不支持）
                  query: nil, # 查询字符串，可以是string或hash类型
                  body: nil, # 请求体，可以是string或hash类型
                  head: nil, # 请求头
@@ -71,9 +71,9 @@ class TaskStruct
 
     @request_options = {
       redirects: redirects,
-      # keepalive: keepalive,
+      keepalive: keepalive,
       file: file,
-      # path: path,
+      path: path,
       query: query,
       body: body,
       head: head
@@ -155,20 +155,20 @@ module ListSpider
 
             if task_struct.errback
               task_struct.errback.call(task_struct, http_req)
-            else
-              ret = false
-              if task_struct.http_method == :get
-                ret = SpiderHelper.direct_http_get(task_struct.href, task_struct.local_path, convert_to_utf8: @convert_to_utf8)
-              elsif task_struct.http_method == :post
-                ret = SpiderHelper.direct_http_post(task_struct.href, task_struct.local_path, task_struct.params, convert_to_utf8: @convert_to_utf8)
-              end
-
-              if ret
-                call_parse_method(task_struct)
-                succeed_list << task_struct
-              else
-                failed_list << task_struct
-              end
+            # else
+            #   ret = false
+            #   if task_struct.http_method == :get
+            #     ret = SpiderHelper.direct_http_get(task_struct.href, task_struct.local_path, convert_to_utf8: @convert_to_utf8)
+            #   elsif task_struct.http_method == :post
+            #     ret = SpiderHelper.direct_http_post(task_struct.href, task_struct.local_path, task_struct.params, convert_to_utf8: @convert_to_utf8)
+            #   end
+
+            #   if ret
+            #     call_parse_method(task_struct)
+            #     succeed_list << task_struct
+            #   else
+            #     failed_list << task_struct
+            #   end
             end
           end
 

data/lib/list_spider/version.rb

@@ -1,3 +1,3 @@
 module ListSpider
-  VERSION = '2.0.0'.freeze
+  VERSION = '2.0.1'.freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: list_spider
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.0.1
 platform: ruby
 authors:
 - Charles Zhang
@@ -108,6 +108,7 @@ files:
 - ".gitignore"
 - ".rubocop.yml"
 - Gemfile
+- Gemfile.lock
 - README.md
 - Rakefile
 - bin/console