tomosia_icon8_crawl 0.1.3 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 49040d5a1bb9faed101a971a31f0c46f57e5500926b11d74c7d017c70a3e5c25
-  data.tar.gz: ac169fdc93fb5a4986c378d6ae7cda1a632bdca2f1dc1b7526b93ab606b3e4ef
+  metadata.gz: 4bb616bc43457255e62c075140fd4ab1b1a462ee1c07fae884221b0f474d9d57
+  data.tar.gz: 899afe155c3484dc7bac9b6df452094b89676ff5d855f865d57710e797248849
 SHA512:
-  metadata.gz: 3bd8d75ab2186da7a999db974bc037354b7179f0ad0846f4782310292a14e3e6bdb5db97dcb8902db8672a197269efb817d75e83d08626cf3bd7b5238572c4a5
-  data.tar.gz: e6916b072fb32ac825c6c42016f3d47eea120dfcc27db0f17281f421b106ba56103c2624e8b1ec0bb79bbfb2f5b34eedc9877ae73c14cfa5744ea0acb7d7e442
+  metadata.gz: 928c1591780c975fb2234287136874a7d59f17985b1f3f4a3f623a04411b3af1e1fb2659d0633e4f9be969c044892539182430163281610adfec5c606a97e6d8
+  data.tar.gz: 2e2b5ff37091fd220e6eb701ed7c413d6f948c0e050fe65fefdda1899e92fdd4544cce86e1daf49b3654ff18c5c623c6f7927264ceb183aab21ccbe161597c25

data/Gemfile

@@ -6,5 +6,6 @@ gemspec
 gem "rake", "~> 12.0"
 gem "rspec", "~> 3.0"
 gem 'writeexcel', '~> 1.0', '>= 1.0.5'
-gem 'pry', '~> 0.13.1'
-gem 'httparty', '~> 0.17.3'
+gem 'httparty', '~> 0.17.3'
+gem 'thor', '~> 1.0', '>= 1.0.1'
+gem 'nokogiri', '~> 1.10', '>= 1.10.10'

data/Gemfile.lock

@@ -1,60 +1,56 @@
-PATH
-  remote: .
-  specs:
-    tomosia_icon8_crawl (0.1.2)
-      httparty
-      nokogiri
-      pry
-      thor
-      writeexcel
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    coderay (1.1.3)
-    diff-lcs (1.4.4)
-    httparty (0.17.3)
-      mime-types (~> 3.0)
-      multi_xml (>= 0.5.2)
-    method_source (1.0.0)
-    mime-types (3.3.1)
-      mime-types-data (~> 3.2015)
-    mime-types-data (3.2020.0512)
-    mini_portile2 (2.4.0)
-    multi_xml (0.6.0)
-    nokogiri (1.10.10-x64-mingw32)
-      mini_portile2 (~> 2.4.0)
-    pry (0.13.1)
-      coderay (~> 1.1)
-      method_source (~> 1.0)
-    rake (12.3.3)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.2)
-      rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.2)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-support (3.9.3)
-    thor (1.0.1)
-    writeexcel (1.0.5)
-
-PLATFORMS
-  x64-mingw32
-
-DEPENDENCIES
-  bundler
-  httparty (~> 0.17.3)
-  pry (~> 0.13.1)
-  rake (~> 12.0)
-  rspec (~> 3.0)
-  tomosia_icon8_crawl!
-  writeexcel (~> 1.0, >= 1.0.5)
-
-BUNDLED WITH
-   2.1.4
+PATH
+  remote: .
+  specs:
+    tomosia_icon8_crawl (0.1.4)
+      httparty
+      nokogiri
+      thor
+      writeexcel
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    coderay (1.1.3)
+    diff-lcs (1.4.4)
+    httparty (0.17.3)
+      mime-types (~> 3.0)
+      multi_xml (>= 0.5.2)
+    method_source (1.0.0)
+    mime-types (3.3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2020.0512)
+    mini_portile2 (2.4.0)
+    multi_xml (0.6.0)
+    nokogiri (1.10.10-x64-mingw32)
+      mini_portile2 (~> 2.4.0)
+    rake (12.3.3)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+    thor (1.0.1)
+    writeexcel (1.0.5)
+
+PLATFORMS
+  x64-mingw32
+
+DEPENDENCIES
+  bundler
+  httparty (~> 0.17.3)
+  pry (~> 0.13.1)
+  rake (~> 12.0)
+  rspec (~> 3.0)
+  tomosia_icon8_crawl!
+  writeexcel (~> 1.0, >= 1.0.5)
+
+BUNDLED WITH
+   2.1.4

data/README.md

@@ -14,49 +14,49 @@ gem 'tomosia_icon8_crawl'
 
 And then execute:
 
-    $ bundle install
+  $ bundle install
 
 Or install it yourself as:
 
-    $ gem install tomosia_icon8_crawl
+  $ gem install tomosia_icon8_crawl
 
 ## Usage
 
 ### Commandline or Terminal:
 
-    Download all photos including pages from icon8.com with keyword you search
+  Download all photos including pages from icon8.com with keyword you search
     
-        $ tomosia_icon8_crawl crawl "keyword" --destination="storage path"
+    $ tomosia_icon8_crawl crawl "keyword" --destination="storage path"
         
-    Download pictures with number you want to get from icon8.com with keyword to find
+  Download pictures with number you want to get from icon8.com with keyword to find
     
-        $ tomosia_icon8_crawl crawl "keyword" --destination="storage path" --max=number
+    $ tomosia_icon8_crawl crawl "keyword" --destination="storage path" --max=number
         
 ### Developers:
 
-    Gemfile:
+  Gemfile:
     
-        $ gem 'tomosia_icon8_crawl'
+    $ gem 'tomosia_icon8_crawl'
         
-    Require:
+  Require:
     
-        $ require 'tomosia_icon8_crawl'
+    $ require 'tomosia_icon8_crawl'
         
-    Download all photos including pages from icon8.com with keyword you search
+  Download all photos including pages from icon8.com with keyword you search
     
-        $ TomosiaIcon8Crawl::CrawlIcon8.crawl("keyword", "destination")
+    $ TomosiaIcon8Crawl::CrawlIcon8.crawl("keyword", "destination")
         
-    Download pictures with number you want to get from icon8.com with keyword to find
+  Download pictures with number you want to get from icon8.com with keyword to find
     
-        $ TomosiaIcon8Crawl::CrawlIcon8.crawl("keyword", "destination", max)
+    $ TomosiaIcon8Crawl::CrawlIcon8.crawl("keyword", "destination", max)
         
-    Help:
-        - keyword is the word used for searching.
-            ex: corona, car, virus,...
-        - destination is the path of the directory where the image was downloaded.
-            ex: E:\download, C:\download, C:\Desktop,...
-        - max max is the number of images you want to download.
-            ex: 100, 10, 5, 1000,...
+  Help:
+    - keyword is the word used for searching.
+        ex: corona, car, virus,...
+    - destination is the path of the directory where the image was downloaded.
+        ex: E:\download, C:\download, C:\Desktop,...
+    - max max is the number of images you want to download.
+        ex: 100, 10, 5, 1000,...
 
 
 

data/lib/tomosia_icon8_crawl.rb

@@ -1,153 +1,155 @@
 module TomosiaIcon8Crawl
-	require 'open-uri'
-	require "HTTParty"
-	require 'pry'
-	require 'WriteExcel'
-	class CrawlIcon8
+  require 'open-uri'
+  require "httparty"
+  require 'writeexcel'
+  class CrawlIcon8
 
-		# get data from website
-		def self.json(key, max)
-			if key == nil
-				p "No data!"
-			else 
-				if max == nil
-					uri = "https://search.icons8.com/api/iconsets/v5/search?term=#{key}&amount=1"
-					page = HTTParty.get(uri)
-					res = page.parsed_response
-					countAll = res['parameters']['countAll']
-					url = "https://search.icons8.com/api/iconsets/v5/search?term=#{key}&amount=#{countAll}"
-				else
-					url = "https://search.icons8.com/api/iconsets/v5/search?term=#{key}&amount=#{max}"
-				end
-			end
-			
-			page_data = HTTParty.get(url)
-			@responses = page_data.parsed_response	
-		end
+    # get data from website
+    def self.json(key, max = nil)
+      if key == nil
+        p "No data!"
+      else 
+        if max == nil
+          urlc = 'https://search.icons8.com/api/iconsets/v5/search?term=' + key + '&amount=1'
+          page = HTTParty.get(urlc)
+          res = page.parsed_response
+          countAll = res['parameters']['countAll']
+          url = 'https://search.icons8.com/api/iconsets/v5/search?term=' + key + '&amount=' + countAll.to_s
+        else
+           url = 'https://search.icons8.com/api/iconsets/v5/search?term=' + key + '&amount=' + max
+        end
+      end
+        
+      page_data = HTTParty.get(url)
+      @responses = page_data.parsed_response  
+    end
 
-		# save file to txt
-		def self.save_file_txt(index, name, url, size, extension)   
-	      	File.open("log_image.txt", "a+") do |f|
-	      		f.write("#{index}. name: #{name} | url: #{url} | size: #{size}Kb | extension: #{extension} \n")
-	      	end
-		end
+    # save file to txt
+    def self.save_file_txt(index, name, url, size, extension)   
+      File.open("log_image.txt", "a+") do |f|
+        f.write("#{index}. name: #{name} | url: #{url} | size: #{size}Kb | extension: #{extension} \n")
+      end
+    end
 
-		# save file to excel
-		def self.save_file_excel( path, data = {})
-			begin
-				des = path + '/export.xls'
-				workbook  = WriteExcel.new(des)
-				format = workbook.add_format
-				format.set_bold()
-				format.set_align('center')
+    # save file to excel
+    def self.save_file_excel( path, data = {})
+      p "Write excel"
+      begin
+        des = path + '/export.xls'
+        workbook  = WriteExcel.new(des)
+        format = workbook.add_format
+        format.set_bold()
+        format.set_align('center')
 
-				data_col = workbook.add_format
-				data_col.set_align('center')
-				format_url = workbook.add_format
-				format_url.set_color('blue')
-				format_url.set_align('center')
+        data_col = workbook.add_format
+        data_col.set_align('center')
+        format_url = workbook.add_format
+        format_url.set_color('blue')
+        format_url.set_align('center')
 
-				worksheet = workbook.add_worksheet
-				
-	    		worksheet.write_string(0, 0, 'STT', format)
-		    	worksheet.write_string(0, 1, 'NAME', format)
-		    	worksheet.write_string(0, 2, 'URL', format)
-		    	worksheet.write_string(0, 3, 'SIZE', format)
-		    	worksheet.write_string(0, 4, 'EXTENSION', format)
+        worksheet = workbook.add_worksheet
+            
+        worksheet.write_string(0, 0, 'STT', format)
+        worksheet.write_string(0, 1, 'NAME', format)
+        worksheet.write_string(0, 2, 'URL', format)
+        worksheet.write_string(0, 3, 'SIZE(byte)', format)
+        worksheet.write_string(0, 4, 'EXTENSION', format)
 
-		    	multi = []
-		    	data.each_with_index do |row, index|
-		    		i = index + 1
-		    		# p i
-		    		# p row
-		    		multi << Thread.new do
-			    		row.each do |key, value|
-			    			# p key
+        multi = []
+        data.each_with_index do |row, index|
+          i = index + 1
+          # p i
+          # p row
+          multi << Thread.new do
+            row.each do |key, value|
+              # p key
 
-				    		worksheet.write_string(i, 0, row['index'], data_col)
-				    		worksheet.write_string(i, 1, row['name'], data_col)
-				    		worksheet.write_url(i, 2, row['url'], format_url)
-				    		worksheet.write_string(i, 3, row['size'], data_col)
-				    		worksheet.write_string(i, 4, row['extension'], data_col)
-			    		end
-			    	end
-		    	end
-		    	multi.each{ |m| m.join }
+              worksheet.write_string(i, 0, row['index'], data_col)
+              worksheet.write_string(i, 1, row['name'], data_col)
+              worksheet.write_url(i, 2, row['url'], format_url)
+              worksheet.write_string(i, 3, row['size'], data_col)
+              worksheet.write_string(i, 4, row['extension'], data_col)
+            end
+          end
+        end
+        multi.each{ |m| m.join }
 
-	    		workbook.close
-			rescue Exception => e
-				p "Can't saved file"
-				p e
-				# break
-			end
-		end
+        workbook.close
+      rescue Exception => e
+        p "Can't saved file"
+        p e
+      end
+      p "Write excel susscess!"
+    end
 
-		# download image
-		def self.download_image(path, img)
-			timeout = 0
-			begin 
-				open(img) do |image|
-					File.open(path, 'wb') do |file|
-						file.write(image.read)
-						@size = image.size
-						# p @size
-					end	
-				end
-			rescue
-				if timeout < 5
-					timeout += 1
-					p "Retry download image"
-					retry
-				end
-			end		
-		end
+    # download image
+    def self.download_image(path, img)
+      timeout = 0
+      begin 
+        open(img) do |image|
+        File.open(path, 'wb') do |file|
+          file.write(image.read)
+            @size = image.size
+          end 
+        end
+      rescue
+        if timeout < 5
+          timeout += 1
+          p "Retry download image"
+          retry
+        end
+      end     
+    end
 
-		# multi download image
-		def self.multi_download_image(path, imgs)
-			begin
-				threads = []
-				@data = []
+    # multi download image
+    def self.multi_download_image(path, imgs)
+      p "Start download image"
+      print "Loadding"
+      begin
+        threads = []
+        @data = []
 
-				imgs.each_with_index do |img, index|
-					# p index
-					title = File.basename(img, '.png')
-					des = path. + "/" + index.to_s + "-" + title + ".png"
-					ext = File.extname(img).delete('.')
-					threads << Thread.new do 
-						download_image(des, img)
-						@size = @size.to_s + "byte"
-						row = {"index" => index, "name" => title, "url" => img, "size" => @size, "extension" => ext}
-						@data.push(row)
-					end
-				end
-				threads.each{ |t| t.join }	
-			rescue Exception => e
-				p "no data"
-				p e
-			end
-		end
+        imgs.each_with_index do |img, index|
+          # p index
+          title = File.basename(img, '.png')
+          des = path. + "/" + index.to_s + "-" + title + ".png"
+          ext = File.extname(img).delete('.')
+          threads << Thread.new do 
+            download_image(des, img)
+            row = {"index" => index, "name" => title, "url" => img, "size" => @size, "extension" => ext}
+            @data.push(row)
+            print "."
+          end
+        end
+        threads.each{ |t| t.join }  
+      rescue Exception => e
+        p "no data"
+        p e
+      end
+      print "\nDownload success!\n"
+    end
 
-		# main
-	    def self.crawl(keyword = nil, path = ".", max = nil)
-	    	begin
-	        	
-				images = []
-				des = ""
-	        	json(keyword, max)
-				@responses['icons'].each_with_index do |item, index| 
+    # main
+    def self.crawl(keyword = nil, path = ".", max = nil)
+      begin
+          
+        images = []
+        des = ""
+        json(keyword, max)
+        @responses['icons'].each_with_index do |item, index| 
 
-					src = "https://img.icons8.com/#{item['platform']}/2x/#{item['commonName']}.png"
+          src = "https://img.icons8.com/#{item['platform']}/2x/#{item['commonName']}.png"
 
-					# add image
-					images.push(src)
-				end
+          # add image
+          images.push(src)
+        end
 
-				multi_download_image(path, images)
-				save_file_excel(path, @data)
-	    	rescue Exception => e
-	    		p "--Runtime error--"
-	    		p e
-	    	end
-	    end
-	end
+        multi_download_image(path, images)
+        save_file_excel(path, @data)
+      rescue Exception => e
+        p "--Runtime error--"
+        p e
+      end
+    end
+  end
 end

data/lib/tomosia_icon8_crawl/version.rb

@@ -1,3 +1,3 @@
 module TomosiaIcon8Crawl
-  VERSION = "0.1.3"
+  VERSION = "0.1.8"
 end

data/tomosia_icon8_crawl.gemspec

@@ -6,19 +6,18 @@ Gem::Specification.new do |spec|
   spec.authors       = ["ninh-tomosia"]
   spec.email         = ["tt.ninh.le@tomosia.com"]
 
-  spec.summary       = %q{tomosia-icon8-crawl using download image from icon8.com}
+  spec.summary       = %q{tomosia-icon8-crawl using download image from icons8.com}
   spec.homepage      = "https://github.com/ninh-tomosia/tomosia-icon8-crawl"
   spec.license       = "MIT"
   spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
 
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
-  spec.add_runtime_dependency "thor"
+  spec.add_runtime_dependency "thor", '~> 1.0', '>= 1.0.1'
   spec.add_development_dependency "rspec"
-  spec.add_runtime_dependency "httparty"
-  spec.add_runtime_dependency "writeexcel"
-  spec.add_runtime_dependency "nokogiri"
-  spec.add_runtime_dependency "pry"
+  spec.add_runtime_dependency "httparty", '~> 0.17.3'
+  spec.add_runtime_dependency "writeexcel", '~> 1.0', '>= 1.0.5'
+  spec.add_runtime_dependency "nokogiri", '~> 1.10', '>= 1.10.10'
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tomosia_icon8_crawl
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.8
 platform: ruby
 authors:
 - ninh-tomosia
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-08-10 00:00:00.000000000 Z
+date: 2020-08-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -42,16 +42,22 @@ dependencies:
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.1
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -70,58 +76,56 @@ dependencies:
   name: httparty
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.17.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.17.3
 - !ruby/object:Gem::Dependency
   name: writeexcel
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.5
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
+        version: '1.10'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: pry
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.10.10
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.10.10
 description: 
 email:
 - tt.ninh.le@tomosia.com
@@ -170,5 +174,5 @@ requirements: []
 rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
-summary: tomosia-icon8-crawl using download image from icon8.com
+summary: tomosia-icon8-crawl using download image from icons8.com
 test_files: []