list_spider 0.3.6 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: cc5222e3c9eff5b6009e41e29f146de7013ad0f0
-  data.tar.gz: 20a2d9c24d07a8fc4c9b61440bd4cabe256ea0f3
+SHA256:
+  metadata.gz: 197035f7521ba4c326c0181c7133afe4c5d7bacfc3246795dc32758dce40da64
+  data.tar.gz: 89d14776f4c041806b6b9e164b31e651d03746c74d83505d5a32c1aeeaa62aa2
 SHA512:
-  metadata.gz: 83cb5db4b531c32cce3ae5bac6cca154f1b43c175c6659a48cf04c86b6353d1a1763a0500426f7f6b2eb370aec66e311af48727f581d087f5677ea3c345afec2
-  data.tar.gz: fd3d2a42c7127b72396c98c189474b7431c5e92f874085c5d6939d765ec8105abf5d58d5ff6b6dc69ca31fb46f5cefb158ce13e8f828749f122b909182933035
+  metadata.gz: a1b38832345203ec036ff4f8e11fba1d92e8ec58674d05ef129784a9e274dcd03ef421fa3db6e38bc38d7bb1cf3c54b7d56cbb321a5340bbe197fe57099ed077
+  data.tar.gz: 43de7e093004c823abb3c51a053869fd294af7fee9f9724c499af572ead7d5ba79d7ab9bb16b2baae1e00a1d198f89fcfbbedc35f57a3a8ed00f7f785d40cbfc

data/.gitignore

@@ -0,0 +1,84 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+# Windows image file caches
+Thumbs.db
+ehthumbs.db
+
+# Folder config file
+Desktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk
+
+rubocopresult
+coolshell

data/.rubocop.yml

@@ -0,0 +1,48 @@
+Metrics/LineLength:
+  Max: 120
+Metrics/MethodLength:
+  Max: 50
+Metrics/ParameterLists:
+  Max: 12
+Metrics/AbcSize:
+  Max: 50
+Metrics/CyclomaticComplexity:
+  Max: 10
+Metrics/PerceivedComplexity:
+  Max: 10
+Style/GuardClause:
+  MinBodyLength: 5
+Style/AsciiComments:
+  Enabled: false
+Style/Documentation:
+  Enabled: false
+Lint/AmbiguousRegexpLiteral:
+  Enabled: false
+Lint/DefEndAlignment:
+  AutoCorrect: true
+Lint/EndAlignment:
+  AutoCorrect: true
+Style/BracesAroundHashParameters:
+  Enabled: false
+Style/ClassAndModuleChildren:
+  Enabled: false
+Style/AutoResourceCleanup:
+  Enabled: true
+Style/CollectionMethods:
+  Enabled: true
+Style/Encoding:
+  Enabled: true
+Style/MethodCalledOnDoEndBlock:
+  Enabled: true
+Layout/MultilineAssignmentLayout:
+  Enabled: true
+Style/OptionHash:
+  Enabled: true
+Style/StringMethods:
+  Enabled: true
+Style/SymbolArray:
+  Enabled: true
+Style/NonNilCheck:
+  IncludeSemanticChanges: true
+Style/Send:
+  Enabled: true

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in list_spider.gemspec
+gemspec

data/README.md

@@ -0,0 +1,186 @@
+# list_spider
+
+A url list spider based on em-http-request.
+
+Many times we only need to spider by url list then parse them and spider again. This is for the purpose.
+
+## Features
+* Duplicate url filtering (based on local path, so you can custom your behavior).
+
+* Convert to UTF-8 support.
+
+* Increased spider support (don't spider exist).
+
+* Customize concurrent number and interval between task.
+
+* Http options support.
+
+## Getting started
+
+    gem install list_spider
+
+## Use like this
+```ruby
+require 'list_spider'
+
+DOWNLOAD_DIR = 'coolshell/'
+
+$next_list = []
+
+def parse_index_item(file_name)
+  content = File.read(file_name)
+  doc = Nokogiri::HTML(content)
+  list_group = doc.css("h2.entry-title")
+  link_list = list_group.css("a")
+
+  link_list.each do |link|
+    href = link['href']
+    local_path = DOWNLOAD_DIR + link.content + ".html"
+    #or you can save them to database for later use
+    $next_list<< TaskStruct.new(href, local_path)
+  end
+end
+
+task_list = []
+task_list << TaskStruct.new('https://coolshell.cn/', DOWNLOAD_DIR + 'index.html', parse_method: method(:parse_index_item))
+
+ListSpider.get_list(task_list)
+ListSpider.get_list($next_list, max: 60)
+
+```
+
+## Or in one step
+```ruby
+require 'list_spider'
+
+DOWNLOAD_DIR = 'coolshell/'
+
+def parse_index_item(file_name)
+
+  content = File.read(file_name)
+  doc = Nokogiri::HTML(content)
+  list_group = doc.css("h2.entry-title")
+  link_list = list_group.css("a")
+
+  link_list.each do |link|
+    href = link['href']
+    local_path = DOWNLOAD_DIR + link.content + ".html"
+    ListSpider.add_task(TaskStruct.new(href, local_path))
+  end
+end
+
+#get_one is a simple function for one taskstruct situation
+ListSpider.get_one(TaskStruct.new(
+  'https://coolshell.cn/',
+  DOWNLOAD_DIR + 'index.html',
+  parse_method: method(:parse_index_item)),
+max: 60)
+
+```
+
+## You can define parse method in four forms
+
+```ruby
+def parse_response(file_name)
+  #...
+end
+
+
+# extra_data is passed by TaskStruct's extra_data param
+
+def parse_response(file_name, extra_data)
+  #...
+end
+
+
+# response_header is a EventMachine::HttpResponseHeader object
+# you can use it like this:
+# response_header.status
+# response_header.cookie
+# response_header['Last-Modified']
+
+def parse_response(file_name, extra_data, response_header)
+  response_header.status
+  response_header['Last-Modified']
+
+  #...
+end
+
+# req is a EventMachine::HttpClientOptions object
+# you can use it like this:
+# req.body
+# req.headers
+# req.uri
+# req.host
+# req.port
+def parse_response(file_name, extra_data, response_header, req)
+  puts req.body
+  puts req.headers
+  puts req.uri
+  puts req.host
+  puts req.port
+
+  #...
+end
+
+```
+
+## And there are many options you can use
+
+```ruby
+TaskStruct.new(href, local_path, http_method: :get, params: {}, extra_data: nil, parse_method: nil, header: nil)
+```
+
+```ruby
+#no concurrent limit (note: only use when list size is small)
+ListSpider.get_list(down_list, interval: 0, max: ListSpider::NO_LIMIT_CONCURRENT)
+
+#sleep random time, often used in site which limit spider
+ListSpider.get_list(down_list, interval: ListSpider::RANDOM_TIME, max: 1)
+
+#set random time range
+ListSpider.get_list(down_list, interval: (1..10), max: 1)
+
+```
+
+###Options below will take effect in the whole program (set them before call get_list)
+
+```ruby
+#set proxy
+ListSpider.set_proxy(proxy_addr, proxy_port, username: nil, password: nil)
+
+#set http header (if TaskStruct has header it will be used priority)
+ListSpider.set_header_option(header_option)
+
+#convert the file encoding to utf-8
+ListSpider.convert_to_utf8 = true
+
+#set connect timeout
+ListSpider.connect_timeout = 2*60
+
+#over write exist file
+ListSpider.overwrite_exist = false
+
+#set redirect depth
+ListSpider.max_redirects = 10
+
+```
+
+## There is a util class to help check or delete unvalid file
+
+```ruby
+FileFilter.delete(CustomConfig::DIR + '*', size_threshold: 300)
+
+FileFilter.check(CustomConfig::DIR + '*', size_threshold: 300)
+
+FileFilter.check_save_result(CustomConfig::DIR + '*', size_threshold: 300)
+
+#params
+FileFilter.delete(dir_pattern, size_threshold: 1000, cust_judge: nil)
+
+FileFilter.check_save_result(dir_pattern, save_file_name: 'filtered_file.txt', size_threshold: 1000, cust_judge: nil)
+```
+
+### License
+
+(MIT License) - Copyright (c) 2016 Charles Zhang

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler/gem_tasks'
+task default: :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'list_spider'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/check_code.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+rubocop -a -D -f simple -o rubocopresult

data/lib/list_spider.rb

@@ -1,3 +1,4 @@
+require 'list_spider/version'
 require 'em-http-request'
 require 'nokogiri'
 require 'fileutils'
@@ -108,7 +109,7 @@ module ListSpider
                        end
                 end
                 succeed_list << e
-              rescue => e
+              rescue StandardError => e
                 puts e
               end
             end
@@ -122,7 +123,7 @@ module ListSpider
             if e.http_method == :get
               ret = SpiderHelper.direct_http_get(e.href, e.local_path, convert_to_utf8: @convert_to_utf8)
             elsif e.http_method == :post
-               ret = SpiderHelper.direct_http_post(e.href, e.local_path, e.params, convert_to_utf8: @convert_to_utf8)
+              ret = SpiderHelper.direct_http_post(e.href, e.local_path, e.params, convert_to_utf8: @convert_to_utf8)
             end
 
             if ret
@@ -134,7 +135,7 @@ module ListSpider
 
           begin
             multi.add e.local_path, w
-          rescue => exception
+          rescue StandardError => exception
             puts exception
             puts e.href
             puts e.local_path
@@ -248,7 +249,7 @@ module ListSpider
     end
 
     def get_list(down_list, interval: DEFAULT_INTERVAL, max: DEFAULT_CONCURRNET_MAX)
-      if interval.is_a?Range
+      if interval.is_a? Range
         @random_time_range = interval
         interval = RANDOM_TIME
       end
@@ -273,7 +274,7 @@ module ListSpider
     end
 
     def add_task(task)
-      if task.is_a?Array
+      if task.is_a? Array
         need_down_list = filter_list(task)
         @down_list += need_down_list
       elsif task.is_a?TaskStruct

data/lib/list_spider/version.rb

@@ -0,0 +1,3 @@
+module ListSpider
+  VERSION = '1.0.0'.freeze
+end

data/lib/spider_helper.rb

@@ -27,7 +27,7 @@ module SpiderHelper
         else
           puts res
         end
-      rescue => e
+      rescue StandardError => e
         puts e.backtrace
         puts e
         false
@@ -59,7 +59,7 @@ module SpiderHelper
         else
           puts res
         end
-      rescue => e
+      rescue StandardError => e
         puts e
         false
       end

data/list_spider.gemspec

@@ -0,0 +1,31 @@
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'list_spider/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'list_spider'
+  spec.version       = ListSpider::VERSION
+  spec.authors       = ['Charles Zhang']
+  spec.email         = ['gis05zc@163.com']
+
+  spec.summary       = 'List Spider'
+  spec.description   = 'A url list spider based on em-http-request.'
+  spec.homepage      = 'https://github.com/chinazhangchao/list_spider'
+  spec.license = 'MIT'
+
+  spec.files =
+    `git ls-files -z`.split("\x0").reject do |f|
+      f.match(%r{^(test|spec|features)/})
+    end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'rake', '~> 10.0'
+
+  spec.add_dependency 'em-http-request', '~> 1.1', '>= 1.1.3'
+  spec.add_dependency 'nokogiri', '~> 1.6', '>= 1.6.7'
+  spec.add_dependency 'rchardet', '~> 1.6', '>= 1.6.1'
+end

data/spider_example.rb

@@ -0,0 +1,29 @@
+require 'list_spider'
+# require File.expand_path('../lib/list_spider', __FILE__)
+
+DOWNLOAD_DIR = 'coolshell/'.freeze
+
+def parse_index_item(file_name)
+  content = File.read(file_name)
+  doc = Nokogiri::HTML(content)
+  list_group = doc.css('h2.entry-title')
+  link_list = list_group.css('a')
+
+  link_list.each do |link|
+    href = link['href']
+    local_path = DOWNLOAD_DIR + link.content + '.html'
+    ListSpider.add_task(TaskStruct.new(href, local_path))
+  end
+end
+
+# ListSpider.convert_to_utf8 = true
+
+# get_one is a simple function for one taskstruct situation
+ListSpider.get_one(
+  TaskStruct.new(
+    'https://coolshell.cn/',
+    DOWNLOAD_DIR + 'index.html',
+    parse_method: method(:parse_index_item)
+  ),
+  max: 60
+)

data/spider_example_2.rb

@@ -0,0 +1,29 @@
+require 'list_spider'
+
+DOWNLOAD_DIR = 'coolshell/'.freeze
+
+@next_list = []
+
+def parse_index_item(file_name)
+  content = File.read(file_name)
+  doc = Nokogiri::HTML(content)
+  list_group = doc.css('h2.entry-title')
+  link_list = list_group.css('a')
+
+  link_list.each do |link|
+    href = link['href']
+    local_path = DOWNLOAD_DIR + link.content + '.html'
+    # or you can save them to database for later use
+    @next_list << TaskStruct.new(href, local_path)
+  end
+end
+
+task_list = []
+task_list << TaskStruct.new(
+  'https://coolshell.cn/',
+  DOWNLOAD_DIR + 'index.html',
+  parse_method: method(:parse_index_item)
+)
+
+ListSpider.get_list(task_list)
+ListSpider.get_list(@next_list, max: 60)

metadata

@@ -1,15 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: list_spider
 version: !ruby/object:Gem::Version
-  version: 0.3.6
+  version: 1.0.0
 platform: ruby
 authors:
 - Charles Zhang
 autorequire: 
-bindir: bin
+bindir: exe
 cert_chain: []
-date: 2017-04-24 00:00:00.000000000 Z
+date: 2018-01-29 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
 - !ruby/object:Gem::Dependency
   name: em-http-request
   requirement: !ruby/object:Gem::Requirement
@@ -71,14 +99,27 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.6.1
 description: A url list spider based on em-http-request.
-email: gis05zc@163.com
+email:
+- gis05zc@163.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitignore"
+- ".rubocop.yml"
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- check_code.sh
 - lib/file_filter.rb
 - lib/list_spider.rb
+- lib/list_spider/version.rb
 - lib/spider_helper.rb
+- list_spider.gemspec
+- spider_example.rb
+- spider_example_2.rb
 homepage: https://github.com/chinazhangchao/list_spider
 licenses:
 - MIT
@@ -99,7 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.6
+rubygems_version: 2.7.3
 signing_key: 
 specification_version: 4
 summary: List Spider