arb-bs 1.1.2 → 1.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5a44373bde98e9a832f25ab705840704aec79fa8
-  data.tar.gz: d9523504a380b8b7fa7b3c7f1bcacf9107b03204
+  metadata.gz: 5b63afe38e81d5cfa92a61fa08eb0e2d0f1d7020
+  data.tar.gz: bd1beab3a6de91d439c340203660bfa021fb6f3c
 SHA512:
-  metadata.gz: 37d22665bb8bbaee7eb840a76a4d74262a6ff56c26f99df631e74d9eb147927467008008c73c0a17af1c1fac62d379a0d6642eca1b1d5931b78ac74572dee864
-  data.tar.gz: f7a77ce4aa1fd1999b72b0ff63c1a22a7887d5315d9fb0283637dfe288c1afa277bba77f10b16df42b79f7416b5241f67cda1f82ca99a037c89813303a64de2e
+  metadata.gz: 9928186d35831a79297e25806c8d81bb3697e986abc520055dd9a574ab09b33f7828d6cbb592e9960f1892d15d17b53e510ad76a8c280d1da862e6bba7db93c7
+  data.tar.gz: 916a0fcf65a183207f9a80d3fb9b084d2ae4ab15c8d693e668f12d060505d0f8144e9149a381847186c64ef6a3a0cfcd97d5d444d0774701efce721e594779d6

data/exe/bs_pic_url

@@ -0,0 +1,51 @@
+#!/usr/bin/env ruby
+
+require 'arb/thread'
+require 'arb/crawler'
+
+include Arb
+
+domain='www.budejie.com'
+map_file='map.txt'
+
+thread_count=(ARGV[0] || 3).to_i
+max_page=(ARGV[1] || 50).to_i
+#Minimun idle time(in seconds) between two complete rounds.
+min_idle_time=(ARGV[2] || 600).to_i
+
+
+File.open(map_file,'w+') unless File.exists? map_file
+
+Thread.parallel(thread_count) do |dispatcher|
+
+  loop do
+    "http://#{domain}/pic/?".enum('?',1..max_page).each_with_index do |url,index|
+      dispatcher.new_task do |lock|
+        res=Crawler.get_by_css(url,"div.j-r-list-c-img a img")
+        unless res
+          puts "Some errors occur when parsing page #{index+1}."
+          next
+        end
+        res.each do |hash|
+          url_file=Crawler.filename_of_url(hash[:"data-original"])
+
+          lock.synchronize do 
+            unless File.readlines(map_file).find{|line| line.to_s.include? url_file}
+                puts "#{hash[:'data-original']}\n#{hash[:title]}",''
+                File.open map_file,'a' do |file|
+                  file.puts "#{hash[:'data-original']}:#{hash[:title]}"
+                end
+            end
+          end
+
+        end
+        tmp=1+rand(5)
+        puts "Page round finished for page #{index+1}, next action in #{tmp} seconds later."
+        sleep tmp
+      end
+    end
+    tmp=min_idle_time+rand(5)
+    puts "Complete round finished, next action in #{tmp} seconds later."
+    sleep tmp
+  end
+end

data/lib/arb/bs/version.rb

@@ -1,5 +1,5 @@
 module Arb
   module Bs
-    VERSION = "1.1.2"
+    VERSION = "1.1.3"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: arb-bs
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.1.3
 platform: ruby
 authors:
 - arybin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-04-11 00:00:00.000000000 Z
+date: 2018-04-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -71,6 +71,7 @@ email:
 - arybin@163.com
 executables:
 - bs_pic
+- bs_pic_url
 extensions: []
 extra_rdoc_files: []
 files:
@@ -82,6 +83,7 @@ files:
 - bin/console
 - bin/setup
 - exe/bs_pic
+- exe/bs_pic_url
 - lib/arb/bs.rb
 - lib/arb/bs/version.rb
 homepage: https://github.com/arybin-cn/arb-bs