tomosia_wallhere_crawl 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b52398c94e10a3e7d0b5efaacf9f51e5c77565d0c053017221abca624d025932
-  data.tar.gz: e8568e2f02a76cea021e43512019b66a554d952819aea064229b02bf3ab2d93f
+  metadata.gz: a6c61da5b1321f6aedfbcd3ba4dba8e5fb101ec4f36ca4e6c974e1f26ab01cf7
+  data.tar.gz: 8b3d0be435727d25a4773a02b57d8f063bba8bb7b1a550724306dedf3003de39
 SHA512:
-  metadata.gz: 336d1c213416c431882c12cdae1296860720466d1e77ca5febe0cc86f182910c37f4a0aa282fdd125ab051b68ffb688e659f62ae42e0988ba36501953d283234
-  data.tar.gz: ea2ffbc8c6760cc615184b7978e1066bb15f2b276bb0935af1db7dfaf68cfaac977818afbb3e09f24f832fdaabb38aa46bb23daf3c4836c1e17209b75d2820ae
+  metadata.gz: 6a5064bca8a988c533b110ed2b31c6f7b264030dfd4fd7ecc1766dec801096bc85c9bf374157b21bd08f2b9fee1ed9743ed7aeea845e8195b85ee30490d3280f
+  data.tar.gz: b07d631fa4d6c33806e91641688cda81f872ebdfa9a77fc7af830eed579aaf9e1315bdf7ce623680263fa531e94bdac0f7ba0af67597880b0e92363f5bd3cbd0

data/Gemfile

@@ -4,3 +4,6 @@ source "https://rubygems.org"
 gemspec
 
 gem "rake", "~> 12.0"
+gem 'nokogiri', '~> 1.10', '>= 1.10.10'
+gem "rspec", "~> 3.0"
+gem 'writeexcel', '~> 1.0', '>= 1.0.5'

data/Gemfile.lock

@@ -0,0 +1,45 @@
+PATH
+  remote: .
+  specs:
+    tomosia_wallhere_crawl (0.1.1)
+      nokogiri
+      thor
+      writeexcel
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.4.4)
+    mini_portile2 (2.4.0)
+    nokogiri (1.10.10)
+      mini_portile2 (~> 2.4.0)
+    rake (12.3.3)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+    thor (1.0.1)
+    writeexcel (1.0.5)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler
+  nokogiri (~> 1.10, >= 1.10.10)
+  rake (~> 12.0)
+  rspec (~> 3.0)
+  tomosia_wallhere_crawl!
+  writeexcel (~> 1.0, >= 1.0.5)
+
+BUNDLED WITH
+   2.1.4

data/README.md

@@ -21,9 +21,17 @@ Or install it yourself as:
     $ gem install tomosia_wallhere_crawl
 
 ## Usage
+_NOTE: key = "tag" , destination: "url saving" , max :number image
 
-TODO: Write usage instructions here
+_/ Crawl all image in tag!
+tomosia_wallhere_crawl crawl "key" --destination " "
+=> tomosia_wallhere_crawl crawl "aaa" --destination "/home/tung/Desktop/img/"
 
+_/ Crawl all images in tag with number of images!
+tomosia_wallhere_crawl crawl "key" --destination " " --max
+=> tomosia_wallhere_crawl crawl "aaa" --destination "/home/tung/Desktop/img/" --max=100
+
+File save info images of tag in destination
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/Rakefile

@@ -1,2 +1,10 @@
 require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
 task :default => :spec
+
+namespace :gem do
+  task :build do
+    system "rake build && rake install"
+  end
+end

data/exe/tomosia_wallhere_crawl

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'tomosia_wallhere_crawl/cli'
+TomosiaWallhereCrawl::Cli.start

data/lib/tomosia_wallhere_crawl.rb

@@ -1,10 +1,12 @@
 require 'open-uri'
 require 'nokogiri'
 require 'writeexcel'
+require 'thor'
 module TomosiaWallhereCrawl
 	class CrawlImage
-		def savedata (data = {}, description)
-		  workbook  = WriteExcel.new("#{description}/InfoImage.xls")
+
+		def savedata (data = {}, destination)
+		  workbook  = WriteExcel.new("#{destination}/InfoImage.xls")
 		  worksheet = workbook.add_worksheet
 		    data.each_with_index do |row, stt|
 		    row.each do |key, value|
@@ -14,13 +16,14 @@ module TomosiaWallhereCrawl
 		      worksheet.write_string(stt, 3, row['size'])
 		      end
 		    end
-		  workbook.close
+			workbook.close
+			puts "Save successfully"
 		end
-		def crawldata(key,description,max)
-			data=[]
-			
+
+		def crawldata(key,destination,max=nil)	
 			  	sum = 0
-			  	index = 1
+					index = 1
+					images = []
 				while sum != max do
 				 	# Open url
 				  	url = "https://wallhere.com/en/wallpapers?q=#{key}&page=#{index}"
@@ -28,45 +31,62 @@ module TomosiaWallhereCrawl
 				  	content = document.read
 				  	parsed_content = Nokogiri::HTML(content)				  
 				  	length = parsed_content.css('.item').to_a.length - 1
-				  	if length == -1
-				  		break
-				  	else
-					  	i = 0
+						total_img = parsed_content.css('div.hub-totalinfo').text.split(' HD Wallpapers')[0].to_i
+						if max == nil || max > total_img
+							max = total_img
+							puts "This tag has #{total_img} pictures"	
+						end
+							i = 0
 					  	for i in i..length
-						    	nameimg = File.basename(parsed_content.css('.item').to_a[i].children.children.first.to_h['src']).delete('.jpg!s')
-							    # Get url img
-							    urlimg = parsed_content.css('.item').to_a[i].children.children.first.to_h['src']
-							    open(urlimg) do |image|
-								    File.open("#{description}#{nameimg}", "w+") do |file|
-								        file.write(image.read)
-								      	#Get info img
-								        n = nameimg
-								        ui = parsed_content.css('.item').to_a[i].children.children.first.to_h['src']
-								        ex = File.extname(parsed_content.css('.item').to_a[i].children.children.first.to_h['src']).delete('.!s')
-								        size = File.size("#{description}#{nameimg}")
-								        s = "#{size} kb"
-								        row = {'stt'=>i, 'name'=>n, 'url'=>ui, 'extension'=>ex, 'size'=>s}
-								        data.push(row)
-								    end
-								end
-								sum += 1
-							    if max == sum
-							      break
-							    end
-					  	end
+									urlimg = parsed_content.css('.item').to_a[i].children.children.first.to_h['src']
+									images.push(urlimg)
+										
+										print '.'
+										sum += 1
+							   	 	if max == sum 
+							      	break
+							end
+						end
+				    	index += 1
 					end
-				    index += 1
+					download(images,destination)
+			end
+
+		def download(images,destination)
+			data = []
+			row = {}
+			thread = []
+			images.each do |img|	
+					thread << Thread.new(img) do
+					timeout = 0
+					begin 
+						open(img) do |image|
+							nameimg = File.basename(img,".jpg!s")
+							ui = img
+							ex = File.extname(img).delete('.!s')
+							size = ""
+							File.open("#{destination}#{nameimg}","wb") do |file|
+								file.write(image.read)
+								size = image.size
+							end
+							size = size.to_s + " bytes"
+							row = {"name"=>nameimg, "url"=>ui, "extension"=> ex, "size" => size}
+							data.push(row)
+						end
+					rescue => exception
+						if timeout < 3
+							timeout += 1
+							retry
+						else
+							next
+						end
+					end	
 				end
-				savedata(data,description)
-    end
-    def crawl(key,description,max = nil)
-      mutithread = (0..2).map do
-        Thread.new do
-          crawldata(key,description,max)
-        end
-      end
-      mutithread.map(&:join)
-    end
+			end
+			thread.each {|t| t.join}
+			puts " "
+			puts "Download successfully"
+			savedata(data,destination)
 	end
 end
-
+end

data/lib/tomosia_wallhere_crawl/cli.rb

@@ -0,0 +1,17 @@
+require 'thor'
+require_relative '../tomosia_wallhere_crawl'
+
+module TomosiaWallhereCrawl 
+  class Cli < Thor
+    desc "crawl KEYWORD", "enter KEYWORD to search"
+    option :destination
+    option :max
+    def crawl(keyword)
+      if options[:max] == nil
+        TomosiaWallhereCrawl::CrawlImage.new.crawldata(keyword, options[:destination], options[:max])
+      else
+        TomosiaWallhereCrawl::CrawlImage.new.crawldata(keyword, options[:destination], options[:max].to_i)
+      end
+    end
+  end
+end

data/lib/tomosia_wallhere_crawl/version.rb

@@ -1,3 +1,3 @@
 module TomosiaWallhereCrawl
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/tomosia_wallhere_crawl.gemspec

@@ -11,12 +11,17 @@ Gem::Specification.new do |spec|
   spec.homepage      = "https://github.com/nguyensontung183183/tomosia_wallhere_crawl.git"
   spec.license       = "MIT"
   spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
-  # Specify which files should be added to the gem when it is released.
-  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  end
+
+
+  spec.files         = `git ls-files`.split("\n")
   spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.executables   = ["tomosia_wallhere_crawl"]
   spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency 'thor'
+  spec.add_runtime_dependency 'nokogiri'
+  spec.add_runtime_dependency 'writeexcel'
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
 end

metadata

@@ -1,31 +1,121 @@
 --- !ruby/object:Gem::Specification
 name: tomosia_wallhere_crawl
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - nguyen son tung
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-08-05 00:00:00.000000000 Z
-dependencies: []
+date: 2020-08-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: writeexcel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Write a longer description or delete this line.
 email:
 - nguyensontung18183
-executables: []
+executables:
+- tomosia_wallhere_crawl
 extensions: []
 extra_rdoc_files: []
 files:
 - CODE_OF_CONDUCT.md
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
 - bin/console
 - bin/setup
+- exe/tomosia_wallhere_crawl
 - lib/tomosia_wallhere_crawl.rb
+- lib/tomosia_wallhere_crawl/cli.rb
 - lib/tomosia_wallhere_crawl/version.rb
+- pkg/tomosia_wallhere_crawl-0.1.1.gem
+- tomosia_wallhere_crawl-0.1.0.gem
 - tomosia_wallhere_crawl.gemspec
 homepage: https://github.com/nguyensontung183183/tomosia_wallhere_crawl.git
 licenses: