arb-bs 1.0.2 → 1.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/exe/bs_pic +10 -6
- data/lib/arb/bs/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5a44373bde98e9a832f25ab705840704aec79fa8
|
4
|
+
data.tar.gz: d9523504a380b8b7fa7b3c7f1bcacf9107b03204
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 37d22665bb8bbaee7eb840a76a4d74262a6ff56c26f99df631e74d9eb147927467008008c73c0a17af1c1fac62d379a0d6642eca1b1d5931b78ac74572dee864
|
7
|
+
data.tar.gz: f7a77ce4aa1fd1999b72b0ff63c1a22a7887d5315d9fb0283637dfe288c1afa277bba77f10b16df42b79f7416b5241f67cda1f82ca99a037c89813303a64de2e
|
data/exe/bs_pic
CHANGED
@@ -20,7 +20,7 @@ Thread.parallel(thread_count) do |dispatcher|
|
|
20
20
|
|
21
21
|
loop do
|
22
22
|
"http://#{domain}/pic/?".enum('?',1..max_page).each_with_index do |url,index|
|
23
|
-
dispatcher.new_task do
|
23
|
+
dispatcher.new_task do |lock|
|
24
24
|
res=Crawler.get_by_css(url,"div.j-r-list-c-img a img")
|
25
25
|
unless res
|
26
26
|
puts "Some errors occur when parsing page #{index+1}."
|
@@ -28,14 +28,18 @@ Thread.parallel(thread_count) do |dispatcher|
|
|
28
28
|
end
|
29
29
|
res.each do |hash|
|
30
30
|
url_file=Crawler.filename_of_url(hash[:"data-original"])
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
31
|
+
|
32
|
+
lock.synchronize do
|
33
|
+
unless File.readlines(map_file).find{|line| line.to_s.include? url_file}
|
34
|
+
if Crawler.download(hash[:"data-original"],url_file)
|
35
|
+
puts "#{hash[:'data-original']}\n#{hash[:title]}",''
|
36
|
+
File.open map_file,'a' do |file|
|
37
|
+
file.puts "#{url_file}:#{Crawler.filter_str(hash[:title])}"
|
38
|
+
end
|
36
39
|
end
|
37
40
|
end
|
38
41
|
end
|
42
|
+
|
39
43
|
end
|
40
44
|
tmp=1+rand(5)
|
41
45
|
puts "Page round finished for page #{index+1}, next action in #{tmp} seconds later."
|
data/lib/arb/bs/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: arb-bs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- arybin
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-04-
|
11
|
+
date: 2018-04-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|