arb-bs 1.0.2 → 1.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 78cd3a7ab49fc17305276dc8adb73796469dbd4a
-  data.tar.gz: fbef3647b36f7d752025a010381a60bfded50398
+  metadata.gz: 5a44373bde98e9a832f25ab705840704aec79fa8
+  data.tar.gz: d9523504a380b8b7fa7b3c7f1bcacf9107b03204
 SHA512:
-  metadata.gz: edfcfc35241a1dd0abc9424d31d92525332bfad07074fe1e5231df7385f1f0d421e18eec835c7bc070fe70dcccf85b0fe88707334a4dd795d32a02a68d85f966
-  data.tar.gz: 1a4982acc65d38a757ace416af9e156ff01ca253363bbbdc64ba82bedc82c0ffd635597bc572b866d3070a5d3bedf0da44856eb28cbb7697fcc62f784c071fd1
+  metadata.gz: 37d22665bb8bbaee7eb840a76a4d74262a6ff56c26f99df631e74d9eb147927467008008c73c0a17af1c1fac62d379a0d6642eca1b1d5931b78ac74572dee864
+  data.tar.gz: f7a77ce4aa1fd1999b72b0ff63c1a22a7887d5315d9fb0283637dfe288c1afa277bba77f10b16df42b79f7416b5241f67cda1f82ca99a037c89813303a64de2e

data/exe/bs_pic

@@ -20,7 +20,7 @@ Thread.parallel(thread_count) do |dispatcher|
 
   loop do
     "http://#{domain}/pic/?".enum('?',1..max_page).each_with_index do |url,index|
-      dispatcher.new_task do
+      dispatcher.new_task do |lock|
         res=Crawler.get_by_css(url,"div.j-r-list-c-img a img")
         unless res
           puts "Some errors occur when parsing page #{index+1}."
@@ -28,14 +28,18 @@ Thread.parallel(thread_count) do |dispatcher|
         end
         res.each do |hash|
           url_file=Crawler.filename_of_url(hash[:"data-original"])
-          unless File.readlines(map_file).find{|line| line.to_s.include? url_file}
-            if Crawler.download(hash[:"data-original"],url_file)
-              puts "#{hash[:'data-original']}\n#{hash[:title]}",''
-              File.open map_file,'a' do |file|
-                file.puts "#{url_file}:#{Crawler.filter_str(hash[:title])}"
+
+          lock.synchronize do 
+            unless File.readlines(map_file).find{|line| line.to_s.include? url_file}
+              if Crawler.download(hash[:"data-original"],url_file)
+                puts "#{hash[:'data-original']}\n#{hash[:title]}",''
+                File.open map_file,'a' do |file|
+                  file.puts "#{url_file}:#{Crawler.filter_str(hash[:title])}"
+                end
               end
             end
           end
+
         end
         tmp=1+rand(5)
         puts "Page round finished for page #{index+1}, next action in #{tmp} seconds later."

data/lib/arb/bs/version.rb

@@ -1,5 +1,5 @@
 module Arb
   module Bs
-    VERSION = "1.0.2"
+    VERSION = "1.1.2"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: arb-bs
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.1.2
 platform: ruby
 authors:
 - arybin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-04-05 00:00:00.000000000 Z
+date: 2018-04-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler