list_spider 2.5.0 → 2.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d58268cff218eb1a17602e842d6050a18fd87b5d975ad3b4a142320c8229d18
-  data.tar.gz: 03a8c6918e05656fe381a17402fd05182cceaad4fbadfeeb5753233a250c1aee
+  metadata.gz: e51274f347602a7a0ef0cc3b8a7f0b809f08168f74056a18f8c61d9583644c8f
+  data.tar.gz: adefb63c47d4084ffb50790716c42e839e0b0652985f3b9bfdaafb0ce0fd98b6
 SHA512:
-  metadata.gz: d12f5bb374abb8fcd0f727f2e2f78909c022e8ed43f720f8e913eae46192ba3cfa45dbd45faeb76f76c57e1725abda674e7282ca3ac56a35ed7ac6b76b47e541
-  data.tar.gz: 324d9f632091ef9301a666e1ed7ffe963cb1647e7ceb33774324cd8c30afa5738305dfb508d1318fee3db15eba0547fc2e7a184d4ed027c7e7bc2340405ea31b
+  metadata.gz: cbff095a5255a8b8b9c9fa23d1c17022e31b84a3bcb4b13f2579aebcb2f968d07a543adb9fb2cd19e082539a08845bdae26b675ead2464d5f45ae859d61f7379
+  data.tar.gz: 141c006a26e5740cfa697bf2f2a05ded10c57576556f4b67dbb6ed437a57dabdfc0d0b1ffb5b8153bd96717cafecf1181d8dfb4af4cb88ce8a76aad8dc5e2f34

data/README.md

@@ -100,11 +100,38 @@ get_list(down_list, interval: DEFAULT_INTERVAL, max: DEFAULT_CONCURRNET_MAX)
 get_one(task, interval: DEFAULT_INTERVAL, max: DEFAULT_CONCURRNET_MAX)
 ```
 
+## 可以设置为不保存文件，用`ListSpider.save_file = false`，此时不会根据文件路径做去重处理
+```
+require 'list_spider'
+
+def call_back(task_struct, http_req)
+  puts "succeed"
+  puts http_req.response_header.status
+  File.write("index.html", http_req.response)
+end
+
+def err_back(task_struct, http_req)
+  puts "failed"
+  puts http_req.response_header.status
+end
+
+ListSpider.save_file = false
+
+# get_one is a simple function for one taskstruct situation
+ListSpider.get_one(
+  TaskStruct.new(
+    'https://coolshell.cn/',
+    callback: method(:call_back),
+    errback: method(:err_back)
+  )
+)
+```
+
 ## 下面是TaskStruct可以设置的选项，与[em-http-request](https://github.com/igrigorik/em-http-request)基本一致
 
 ```ruby
 new(href, # 请求链接
-    local_path, # 保存数据的本地路径（此路径作为去重标准）
+    local_path = :nil, # 保存数据的本地路径（保存文件的情况下此路径作为去重标准）
     # http方法，取值：:get, :head, :delete, :put, :post, :patch, :options
     http_method: :get,
     custom_data: nil, # 自定义数据
@@ -167,6 +194,8 @@ end
 def call_back(task_struct, http_req)
   # http_req 是EventMachine::HttpRequest对象
   # http_req.response_header.status
+  # http_req.response_header
+  # http_req.response
   # ...
 end
 

data/lib/list_spider.rb

@@ -3,6 +3,7 @@ require 'em-http-request'
 require 'nokogiri'
 require 'fileutils'
 require 'set'
+require 'securerandom'
 require 'addressable/uri'
 require File.expand_path('spider_helper', __dir__)
 require File.expand_path('file_filter', __dir__)
@@ -10,12 +11,12 @@ require File.expand_path('file_filter', __dir__)
 # 爬取任务类
 class TaskStruct
   # * href 请求链接
-  # * local_path 保存数据的本地路径（此路径作为去重标准）
+  # * local_path 保存数据的本地路径（保存文件的情况下此路径作为去重标准）
   # * http_method http方法，取值：:get, :head, :delete, :put, :post, :patch, :options
   # * custom_data 自定义数据
   # * parse_method 解析保存文件的回调，参数是TaskStruct对象本身
   def initialize(href, # 请求链接
-                 local_path, # 保存数据的本地路径（此路径作为去重标准）
+                 local_path = :nil, # 保存数据的本地路径（保存文件的情况下此路径作为去重标准）
                  # http方法，取值：:get, :head, :delete, :put, :post, :patch, :options
                  http_method: :get,
                  custom_data: nil, # 自定义数据
@@ -117,8 +118,11 @@ module ListSpider
   @random_time_range = 3..10
   @local_path_set = Set.new
   @down_list = []
+  @save_file = true
 
   class << self
+    attr_accessor :save_file
+
     def get_list(down_list, interval: DEFAULT_INTERVAL, max: DEFAULT_CONCURRNET_MAX)
       if interval.is_a? Range
         @random_time_range = interval
@@ -170,7 +174,7 @@ module ListSpider
             s = http_req.response_header.status
             puts "#{Time.now}, http status code: #{s}"
 
-            if s == 200
+            if s == 200 && @save_file
               local_dir = File.dirname(task_struct.local_path)
               FileUtils.mkdir_p(local_dir) unless Dir.exist?(local_dir)
               begin
@@ -197,7 +201,11 @@ module ListSpider
           end
 
           begin
-            multi.add task_struct.local_path, http_req
+            if @save_file
+              multi.add task_struct.local_path, http_req
+            else
+              multi.add SecureRandom.uuid, http_req
+            end
           rescue StandardError => exception
             puts exception
             puts task_struct.href
@@ -279,6 +287,8 @@ module ListSpider
     end
 
     def filter_list(down_list)
+      return down_list unless @save_file
+      
       need_down_list = []
       down_list.each do |ts|
         if !ts.overwrite_exist && File.exist?(ts.local_path)

data/lib/list_spider/version.rb

@@ -1,3 +1,3 @@
 module ListSpider
-  VERSION = '2.5.0'.freeze
+  VERSION = '2.6.0'.freeze
 end

data/spider_not_save_file.rb

@@ -0,0 +1,34 @@
+require 'list_spider'
+# require File.expand_path('../lib/list_spider', __FILE__)
+
+def call_back(task_struct, http_req)
+  puts "succeed"
+  puts http_req.response_header.status
+  content = http_req.response
+  doc = Nokogiri::HTML(content)
+  list_group = doc.css('h2.entry-title')
+  link_list = list_group.css('a')
+
+  link_list.each do |link|
+    href = link['href']
+    ListSpider.add_task(TaskStruct.new(href,
+      callback: method(:call_back),
+      errback: method(:err_back)))
+  end
+end
+
+def err_back(task_struct, http_req)
+  puts "failed"
+  puts http_req.response_header.status
+end
+
+ListSpider.save_file = false
+
+# get_one is a simple function for one taskstruct situation
+ListSpider.get_one(
+  TaskStruct.new(
+    'https://coolshell.cn/',
+    callback: method(:call_back),
+    errback: method(:err_back)
+  )
+)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: list_spider
 version: !ruby/object:Gem::Version
-  version: 2.5.0
+  version: 2.6.0
 platform: ruby
 authors:
 - Charles Zhang
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-06 00:00:00.000000000 Z
+date: 2020-04-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -117,6 +117,7 @@ files:
 - list_spider.gemspec
 - spider_example.rb
 - spider_example_2.rb
+- spider_not_save_file.rb
 homepage: https://github.com/chinazhangchao/list_spider
 licenses:
 - MIT