arb-bs 0.1.6 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cc17e9bd17a4771edc4696a6e8a052da7edd1484
-  data.tar.gz: 8038b6face47fd2f01172dbb0577d0e95c551dbf
+  metadata.gz: d3fee1fcde122cff7050d84adeee0089b80a7b59
+  data.tar.gz: f54a5e8778a5718c7c88fb9b4f336bbf0ec1f423
 SHA512:
-  metadata.gz: 57b740f6b723bef1202f1180e6209bcc5093f682539b1d053638c5d11e4180f115142d3e49681adb0707597a75abcc3c137117ee98dedced0498eba19793e1bd
-  data.tar.gz: 20550beb04745265ad363d5f75c66a25adcfec0b2de4c1625302187fe8b8e0fe466a1a890a1ad535b840d9cdcf048cdd0e5031ba65e418ebf1350a2e85964ae9
+  metadata.gz: a915ebb6fe64cf5579683cf1b32082cae589758534b423b9cfce73ef9c70c18f715d442f62283655497dfa9b2c404c88bf57269fdac44c9dd8c8ad0957ad434e
+  data.tar.gz: 36f3da886a3eb1cff62adbd87aee7def07da9e37a8b62c8f3005ba6023a4444dc0d92ed3b049e4bd442108ed4f149e09051356e9fce551f11473c011cb24bbac

data/arb-bs.gemspec

@@ -23,5 +23,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.14"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_dependency "arb-crawler"
+  spec.add_dependency "arb-thread"
 
 end

data/exe/bs_pic

@@ -1,5 +1,6 @@
 #!/usr/bin/env ruby
 
+require 'arb/thread'
 require 'arb/crawler'
 
 include Arb
@@ -9,31 +10,37 @@ map_file='map.txt'
 max_page=(ARGV[0] || 50).to_i
 #Minimun idle time(in seconds) between two complete rounds.
 min_idle_time=(ARGV[1] || 600).to_i
+thread_count=(ARGV[2] || 3).to_i
 
 
 File.open(map_file,'w+') unless File.exists? map_file
 
-loop do
-  "http://#{domain}/pic/?".enum('?',1..max_page).each_with_index do |url,index|
-    res=Crawler.get_by_css(url,"div.j-r-list-c-img a img")
-    unless res
-      puts "Some errors occur when parsing page #{index+1}."
-      next
-    end
-    res.each do |hash|
-      url_file=Crawler.filename_of_url(hash[:"data-original"])
-      unless File.readlines(map_file).find{|line| line.to_s.include? url_file}
-        if Crawler.download(hash[:"data-original"],url_file)
-          puts "#{hash[:'data-original']}\n#{hash[:title]}",''
-          File.open map_file,'a' do |file|
-            file.puts "#{url_file}:#{Crawler.filter_str(hash[:title])}"
+Thread.parallel(thread_count) do |dispatcher|
+
+  loop do
+    "http://#{domain}/pic/?".enum('?',1..max_page).each_with_index do |url,index|
+      dispatcher.new_task do
+        res=Crawler.get_by_css(url,"div.j-r-list-c-img a img")
+        unless res
+          puts "Some errors occur when parsing page #{index+1}."
+          next
+        end
+        res.each do |hash|
+          url_file=Crawler.filename_of_url(hash[:"data-original"])
+          unless File.readlines(map_file).find{|line| line.to_s.include? url_file}
+            if Crawler.download(hash[:"data-original"],url_file)
+              puts "#{hash[:'data-original']}\n#{hash[:title]}",''
+              File.open map_file,'a' do |file|
+                file.puts "#{url_file}:#{Crawler.filter_str(hash[:title])}"
+              end
+            end
           end
         end
+        tmp=1+rand(5)
+        puts "Page round finished for page #{index+1}, next action in #{tmp} seconds later."
+        sleep tmp
       end
     end
-    tmp=1+rand(5)
-    puts "Page round finished for page #{index+1}, next action in #{tmp} seconds later."
-    sleep tmp
   end
   tmp=min_idle_time+rand(5)
   puts "Complete round finished, next action in #{tmp} seconds later."

data/lib/arb/bs/version.rb

@@ -1,5 +1,5 @@
 module Arb
   module Bs
-    VERSION = "0.1.6"
+    VERSION = "1.0.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: arb-bs
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 1.0.0
 platform: ruby
 authors:
 - arybin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-02-18 00:00:00.000000000 Z
+date: 2018-04-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: arb-thread
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A demo of Web Crawler using arb-crawler
 email:
 - arybin@163.com
@@ -89,7 +103,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.6.14
 signing_key: 
 specification_version: 4
 summary: A demo of Web Crawler using arb-crawler