list_spider 2.2.0 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 39600b837bb18841d083c7b50dbaadf82e72c3013f690129af6786efec193a39
-  data.tar.gz: 4128e673c551e3fcc2c1f9d4a3302407bcf7bc26829a4957d04ebc0505d5ce07
+  metadata.gz: d2fe3c4889a58bacda6fda400b4156c2adf4fbe774bdf9f15596958a1909d890
+  data.tar.gz: db682834ba36559dfc4bbcb704151b4eddea315b614d1c0ef9aa9d170aa69f58
 SHA512:
-  metadata.gz: f900e8f76086f37239872d9b4452f5d735799100879ac16570d29c9570837adca52c3c9e37c725913920a68add7784bc2f94e2cef42663c54930ae5b3e37ec50
-  data.tar.gz: 90495a4dae2552c3f41e55f0efa61fef0511581eb2e13d90256e0a585c48f7fdb2af167cd8c6daa98ca80c2229d970187fbcec3db4a8edd43738f76f79c18951
+  metadata.gz: a259eb91eda3257d3be3df839274a4d40a3c11a4dc2fa0b3a57e580cef6aa180f06d27b05a2b2ee30ceb0e4bcbf037f55e36351d2034ae41f6636dcadaa14aa3
+  data.tar.gz: 9d2062238acf39de5fd8c86d47eb7106640b31ebf87ff42de245a6e911a2efb3e54e7d4592ede162eb248cc33b59e00482781c023a6b1c739b65b703bc8a39e5

data/.gitignore

@@ -1,84 +1,84 @@
-*.gem
-*.rbc
-/.config
-/coverage/
-/InstalledFiles
-/pkg/
-/spec/reports/
-/spec/examples.txt
-/test/tmp/
-/test/version_tmp/
-/tmp/
-
-## Specific to RubyMotion:
-.dat*
-.repl_history
-build/
-
-## Documentation cache and generated files:
-/.yardoc/
-/_yardoc/
-/doc/
-/rdoc/
-
-## Environment normalisation:
-/.bundle/
-/vendor/bundle
-/lib/bundler/man/
-
-# for a library or gem, you might want to ignore these files since the code is
-# intended to run in multiple environments; otherwise, check them in:
-# Gemfile.lock
-# .ruby-version
-# .ruby-gemset
-
-# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
-.rvmrc
-
-.DS_Store
-.AppleDouble
-.LSOverride
-
-# Icon must end with two \r
-Icon
-
-
-# Thumbnails
-._*
-
-# Files that might appear in the root of a volume
-.DocumentRevisions-V100
-.fseventsd
-.Spotlight-V100
-.TemporaryItems
-.Trashes
-.VolumeIcon.icns
-
-# Directories potentially created on remote AFP share
-.AppleDB
-.AppleDesktop
-Network Trash Folder
-Temporary Items
-.apdisk
-
-# Windows image file caches
-Thumbs.db
-ehthumbs.db
-
-# Folder config file
-Desktop.ini
-
-# Recycle Bin used on file shares
-$RECYCLE.BIN/
-
-# Windows Installer files
-*.cab
-*.msi
-*.msm
-*.msp
-
-# Windows shortcuts
-*.lnk
-
-rubocopresult
-coolshell
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+# Windows image file caches
+Thumbs.db
+ehthumbs.db
+
+# Folder config file
+Desktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk
+
+rubocopresult
+coolshell

data/.rdoc_options

@@ -1,23 +1,23 @@
---- !ruby/object:RDoc::Options
-encoding: UTF-8
-static_path: []
-rdoc_include:
-- "."
-- "/Users/zhangchao/github/list_spider"
-charset: UTF-8
-exclude: 
-hyperlink_all: false
-line_numbers: false
-locale: 
-locale_dir: locale
-locale_name: 
-main_page: 
-markup: markdown
-output_decoration: true
-page_dir: 
-show_hash: false
-tab_width: 8
-template_stylesheets: []
-title: 
-visibility: :protected
-webcvs: 
+--- !ruby/object:RDoc::Options
+encoding: UTF-8
+static_path: []
+rdoc_include:
+- "."
+- "/Users/zhangchao/github/list_spider"
+charset: UTF-8
+exclude: 
+hyperlink_all: false
+line_numbers: false
+locale: 
+locale_dir: locale
+locale_name: 
+main_page: 
+markup: markdown
+output_decoration: true
+page_dir: 
+show_hash: false
+tab_width: 8
+template_stylesheets: []
+title: 
+visibility: :protected
+webcvs: 

data/.rubocop.yml

@@ -1,48 +1,48 @@
-Metrics/LineLength:
-  Max: 120
-Metrics/MethodLength:
-  Max: 50
-Metrics/ParameterLists:
-  Max: 12
-Metrics/AbcSize:
-  Max: 50
-Metrics/CyclomaticComplexity:
-  Max: 10
-Metrics/PerceivedComplexity:
-  Max: 10
-Style/GuardClause:
-  MinBodyLength: 5
-Style/AsciiComments:
-  Enabled: false
-Style/Documentation:
-  Enabled: false
-Lint/AmbiguousRegexpLiteral:
-  Enabled: false
-Layout/DefEndAlignment:
-  AutoCorrect: true
-Layout/EndAlignment:
-  AutoCorrect: true
-Style/BracesAroundHashParameters:
-  Enabled: false
-Style/ClassAndModuleChildren:
-  Enabled: false
-Style/AutoResourceCleanup:
-  Enabled: true
-Style/CollectionMethods:
-  Enabled: true
-Style/Encoding:
-  Enabled: true
-Style/MethodCalledOnDoEndBlock:
-  Enabled: true
-Layout/MultilineAssignmentLayout:
-  Enabled: true
-Style/OptionHash:
-  Enabled: true
-Style/StringMethods:
-  Enabled: true
-Style/SymbolArray:
-  Enabled: true
-Style/NonNilCheck:
-  IncludeSemanticChanges: true
-Style/Send:
-  Enabled: true
+Metrics/LineLength:
+  Max: 120
+Metrics/MethodLength:
+  Max: 50
+Metrics/ParameterLists:
+  Max: 12
+Metrics/AbcSize:
+  Max: 50
+Metrics/CyclomaticComplexity:
+  Max: 10
+Metrics/PerceivedComplexity:
+  Max: 10
+Style/GuardClause:
+  MinBodyLength: 5
+Style/AsciiComments:
+  Enabled: false
+Style/Documentation:
+  Enabled: false
+Lint/AmbiguousRegexpLiteral:
+  Enabled: false
+Layout/DefEndAlignment:
+  AutoCorrect: true
+Layout/EndAlignment:
+  AutoCorrect: true
+Style/BracesAroundHashParameters:
+  Enabled: false
+Style/ClassAndModuleChildren:
+  Enabled: false
+Style/AutoResourceCleanup:
+  Enabled: true
+Style/CollectionMethods:
+  Enabled: true
+Style/Encoding:
+  Enabled: true
+Style/MethodCalledOnDoEndBlock:
+  Enabled: true
+Layout/MultilineAssignmentLayout:
+  Enabled: true
+Style/OptionHash:
+  Enabled: true
+Style/StringMethods:
+  Enabled: true
+Style/SymbolArray:
+  Enabled: true
+Style/NonNilCheck:
+  IncludeSemanticChanges: true
+Style/Send:
+  Enabled: true

data/English_README.md

@@ -1,169 +1,169 @@
-# list_spider
-
-A url list spider based on em-http-request.
-
-Many times we only need to spider by url list then parse them and spider again. This is for the purpose.
-
-## Features
-* Duplicate url filtering (based on local path, so you can custom your behavior).
-
-* Convert to UTF-8 support.
-
-* Increased spider support (don't spider exist).
-
-* Customize concurrent number and interval between task.
-
-* Http options support.
-
-## Getting started
-
-```ruby
-gem install list_spider
-```
-
-Or add it to your Gemfile
-
-```ruby
-gem 'list_spider'
-```
-
-## Use like this
-```ruby
-require 'list_spider'
-
-DOWNLOAD_DIR = 'coolshell/'.freeze
-
-@next_list = []
-
-def parse_index_item(e)
-  content = File.read(e.local_path)
-  doc = Nokogiri::HTML(content)
-  list_group = doc.css('h2.entry-title')
-  link_list = list_group.css('a')
-
-  link_list.each do |link|
-    href = link['href']
-    local_path = DOWNLOAD_DIR + link.content + '.html'
-    # or you can save them to database for later use
-    @next_list << TaskStruct.new(href, local_path)
-  end
-end
-
-task_list = []
-task_list << TaskStruct.new(
-  'https://coolshell.cn/',
-  DOWNLOAD_DIR + 'index.html',
-  parse_method: method(:parse_index_item)
-)
-
-ListSpider.get_list(task_list)
-ListSpider.get_list(@next_list, max: 60)
-```
-
-## Or in one step
-```ruby
-require 'list_spider'
-
-DOWNLOAD_DIR = 'coolshell/'.freeze
-
-def parse_index_item(e)
-  content = File.read(e.local_path)
-  doc = Nokogiri::HTML(content)
-  list_group = doc.css('h2.entry-title')
-  link_list = list_group.css('a')
-
-  link_list.each do |link|
-    href = link['href']
-    local_path = DOWNLOAD_DIR + link.content + '.html'
-    ListSpider.add_task(TaskStruct.new(href, local_path))
-  end
-end
-
-# get_one is a simple function for one taskstruct situation
-ListSpider.get_one(
-  TaskStruct.new(
-    'https://coolshell.cn/',
-    DOWNLOAD_DIR + 'index.html',
-    parse_method: method(:parse_index_item)
-  ),
-  max: 60
-)
-```
-
-## And there are many options you can use
-
-```ruby
-def initialize(href, # 请求链接
-                 local_path, # 保存数据的本地路径（此路径作为去重标准）
-                 # http方法，取值：:get, :head, :delete, :put, :post, :patch, :options
-                 http_method: :get,
-                 custom_data: nil, # 自定义数据
-                 parse_method: nil, # 解析保存文件的回调，参数是TaskStruct对象本身
-                 # 请求成功后的回调，此时可能没有保存文件，比如301，404
-                 # 参数是TaskStruct对象本身和对应的EventMachine::HttpRequest对象
-                 # http.response_header.status 状态码
-                 # http.response_header  返回头
-                 # http.response 返回体
-                 callback: nil,
-                 # 请求失败后的回调
-                 # 参数是TaskStruct对象本身和对应的EventMachine::HttpRequest对象
-                 errback: nil,
-                 stream_callback: nil, # 流数据处理回调
-                 convert_to_utf8: false, # 是否转换为utf8编码
-                 overwrite_exist: false, # 是否覆盖现有文件
-                 # request options
-                 redirects: 3, # 重定向次数
-                 keepalive: nil, # （暂不支持复用）
-                 file: nil, # 要上传的文件路径
-                 path: nil, # 请求路径，在流水线方式请求时有用（暂不支持）
-                 query: nil, # 查询字符串，可以是string或hash类型
-                 body: nil, # 请求体，可以是string或hash类型
-                 head: nil, # 请求头
-                 # connection options
-                 connect_timeout: 60, # 连接超时时间
-                 inactivity_timeout: nil, # 连接后超时时间
-                 # ssl设置
-                 # ssl: {
-                 #     :private_key_file => '/tmp/server.key',
-                 #     :cert_chain_file => '/tmp/server.crt',
-                 #     :verify_peer => false
-                 # }
-                 ssl: nil,
-                 # bind: {
-                 #     :host => '123.123.123.123',   # use a specific interface for outbound request
-                 #     :port => '123'
-                 # }
-                 bind: nil,
-                 # 代理设置
-                 # proxy: {
-                 #     :host => '127.0.0.1',    # proxy address
-                 #     :port => 9000,           # proxy port
-                 #     :type => :socks5         # default proxy mode is HTTP proxy, change to :socks5 if required
-
-                 #     :authorization => ['user', 'pass']  # proxy authorization header
-                 # }
-                 proxy: nil)
-```
-
-## Callback methods form
-
-```ruby
-# called when the file is saved successfully
-def parse_eresponse(task_struct)
-  # ...
-end
-
-def call_back(task_struct, http_req)
-  # http_req is a EventMachine::HttpRequest object
-  # http_req.response_header.status
-  # ...
-end
-
-def err_back(task_struct, http_req)
-  # ...
-end
-```
-
-### License
-
-(MIT License) - Copyright (c) 2016 Charles Zhang
+# list_spider
+
+A url list spider based on em-http-request.
+
+Many times we only need to spider by url list then parse them and spider again. This is for the purpose.
+
+## Features
+* Duplicate url filtering (based on local path, so you can custom your behavior).
+
+* Convert to UTF-8 support.
+
+* Increased spider support (don't spider exist).
+
+* Customize concurrent number and interval between task.
+
+* Http options support.
+
+## Getting started
+
+```ruby
+gem install list_spider
+```
+
+Or add it to your Gemfile
+
+```ruby
+gem 'list_spider'
+```
+
+## Use like this
+```ruby
+require 'list_spider'
+
+DOWNLOAD_DIR = 'coolshell/'.freeze
+
+@next_list = []
+
+def parse_index_item(e)
+  content = File.read(e.local_path)
+  doc = Nokogiri::HTML(content)
+  list_group = doc.css('h2.entry-title')
+  link_list = list_group.css('a')
+
+  link_list.each do |link|
+    href = link['href']
+    local_path = DOWNLOAD_DIR + link.content + '.html'
+    # or you can save them to database for later use
+    @next_list << TaskStruct.new(href, local_path)
+  end
+end
+
+task_list = []
+task_list << TaskStruct.new(
+  'https://coolshell.cn/',
+  DOWNLOAD_DIR + 'index.html',
+  parse_method: method(:parse_index_item)
+)
+
+ListSpider.get_list(task_list)
+ListSpider.get_list(@next_list, max: 60)
+```
+
+## Or in one step
+```ruby
+require 'list_spider'
+
+DOWNLOAD_DIR = 'coolshell/'.freeze
+
+def parse_index_item(e)
+  content = File.read(e.local_path)
+  doc = Nokogiri::HTML(content)
+  list_group = doc.css('h2.entry-title')
+  link_list = list_group.css('a')
+
+  link_list.each do |link|
+    href = link['href']
+    local_path = DOWNLOAD_DIR + link.content + '.html'
+    ListSpider.add_task(TaskStruct.new(href, local_path))
+  end
+end
+
+# get_one is a simple function for one taskstruct situation
+ListSpider.get_one(
+  TaskStruct.new(
+    'https://coolshell.cn/',
+    DOWNLOAD_DIR + 'index.html',
+    parse_method: method(:parse_index_item)
+  ),
+  max: 60
+)
+```
+
+## And there are many options you can use
+
+```ruby
+def initialize(href, # 请求链接
+                 local_path, # 保存数据的本地路径（此路径作为去重标准）
+                 # http方法，取值：:get, :head, :delete, :put, :post, :patch, :options
+                 http_method: :get,
+                 custom_data: nil, # 自定义数据
+                 parse_method: nil, # 解析保存文件的回调，参数是TaskStruct对象本身
+                 # 请求成功后的回调，此时可能没有保存文件，比如301，404
+                 # 参数是TaskStruct对象本身和对应的EventMachine::HttpRequest对象
+                 # http.response_header.status 状态码
+                 # http.response_header  返回头
+                 # http.response 返回体
+                 callback: nil,
+                 # 请求失败后的回调
+                 # 参数是TaskStruct对象本身和对应的EventMachine::HttpRequest对象
+                 errback: nil,
+                 stream_callback: nil, # 流数据处理回调
+                 convert_to_utf8: false, # 是否转换为utf8编码
+                 overwrite_exist: false, # 是否覆盖现有文件
+                 # request options
+                 redirects: 3, # 重定向次数
+                 keepalive: nil, # （暂不支持复用）
+                 file: nil, # 要上传的文件路径
+                 path: nil, # 请求路径，在流水线方式请求时有用（暂不支持）
+                 query: nil, # 查询字符串，可以是string或hash类型
+                 body: nil, # 请求体，可以是string或hash类型
+                 head: nil, # 请求头
+                 # connection options
+                 connect_timeout: 60, # 连接超时时间
+                 inactivity_timeout: nil, # 连接后超时时间
+                 # ssl设置
+                 # ssl: {
+                 #     :private_key_file => '/tmp/server.key',
+                 #     :cert_chain_file => '/tmp/server.crt',
+                 #     :verify_peer => false
+                 # }
+                 ssl: nil,
+                 # bind: {
+                 #     :host => '123.123.123.123',   # use a specific interface for outbound request
+                 #     :port => '123'
+                 # }
+                 bind: nil,
+                 # 代理设置
+                 # proxy: {
+                 #     :host => '127.0.0.1',    # proxy address
+                 #     :port => 9000,           # proxy port
+                 #     :type => :socks5         # default proxy mode is HTTP proxy, change to :socks5 if required
+
+                 #     :authorization => ['user', 'pass']  # proxy authorization header
+                 # }
+                 proxy: nil)
+```
+
+## Callback methods form
+
+```ruby
+# called when the file is saved successfully
+def parse_eresponse(task_struct)
+  # ...
+end
+
+def call_back(task_struct, http_req)
+  # http_req is a EventMachine::HttpRequest object
+  # http_req.response_header.status
+  # ...
+end
+
+def err_back(task_struct, http_req)
+  # ...
+end
+```
+
+### License
+
+(MIT License) - Copyright (c) 2016 Charles Zhang