tomosia_123rf_crawl 0.1.0 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1f78b6e2908eae0a4cbdb18a6004a0cdd1bf85665aeb8c3a56efc9fb66cd0434
-  data.tar.gz: 7e2b5142e956f9978f9fabac6d32824378003f2a45e5131554560831c2f2ad3f
+  metadata.gz: d01cc2c7974a1e7504f55c1343e24e145eff9e953f2a58103db081ee41b4e4a3
+  data.tar.gz: 537d07742bb26609dff324b968eec2983a391ba25e318a84410392e34e76f02f
 SHA512:
-  metadata.gz: f55f661cded1ff753eb0d13069266d9c81e9f3f5d8dc0ace6d9a9d5aacf7e287e3e81d3871de9fe4ca978ee1893c70d56f088eb80dc0fdcae012b78f0a3bffee
-  data.tar.gz: 461112743f48a7e37e25e16beb4bef9c6251b5c442cd283e829fb543f9c9f9eac82bec80fb23244df25b7741f5ee8902905c3e02862ae32146c58ee71c11614f
+  metadata.gz: d49573e81fd948f6bb2464acb9e5b214d9ccab3e0e1b221ee8ea7767b4bc95d692f84b3ddd22259e1c33b6244faf4febd0f3d8fe30398e7e287ce2f91e364c73
+  data.tar.gz: 87f9360b304fa785a0fbcda0d42a78f73aa9cfc41e337491f8c0b54875d6c8ed354235ca147c460fe7a8acce402d8252d038f54bd9d5b48d21432448e72a8e48

data/Gemfile

@@ -4,6 +4,3 @@ source "https://rubygems.org"
 gemspec
 
 gem "rake", "~> 12.0"
-gem 'httparty'
-gem 'nokogiri'
-

data/Gemfile.lock

@@ -0,0 +1,37 @@
+PATH
+  remote: .
+  specs:
+    tomosia_123rf_crawl (0.1.9)
+      httparty (= 0.18.1)
+      nokogiri (= 1.10.10)
+      spreadsheet (= 1.2.6)
+      thor
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    httparty (0.18.1)
+      mime-types (~> 3.0)
+      multi_xml (>= 0.5.2)
+    mime-types (3.3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2020.0512)
+    mini_portile2 (2.4.0)
+    multi_xml (0.6.0)
+    nokogiri (1.10.10)
+      mini_portile2 (~> 2.4.0)
+    rake (12.3.3)
+    ruby-ole (1.2.12.2)
+    spreadsheet (1.2.6)
+      ruby-ole (>= 1.0)
+    thor (1.0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rake (~> 12.0)
+  tomosia_123rf_crawl!
+
+BUNDLED WITH
+   2.1.4

data/Rakefile

@@ -1,2 +1,12 @@
 require "bundler/gem_tasks"
+# require "rspec/core/rake task"
+# Rspec::Core::RakeTask.new(:spec)
+
 task :default => :spec
+
+namespace :gem do
+    task :build do
+        system "rake build && rake install"
+    end
+end
+

data/exe/tomosia_123rf_crawl

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require 'tomosia_123rf_crawl/cli'
+Tomosia123rfCrawl::Cli.start

data/lib/tomosia_123rf_crawl.rb

@@ -1,11 +1,93 @@
 require "tomosia_123rf_crawl/version"
-require "tomosia_123rf_crawl/tomosia_123rf_crawl"
 
-module ActionView
-  module Helpers
-    module AssetTagHelper
-      include Tomosia123rfCrawl
+module Tomosia123rfCrawl
+  require 'nokogiri'
+  require 'open-uri'
+  require 'fileutils'
+  require 'spreadsheet' 
+
+  class Crawler
+
+    #Hàm dùng để download các ảnh từ web về
+    def downloadImages(images, destination)
+      #Tạo forder dùng để lưu hình ảnh
+      path= "#{destination}/images" 
+
+    #Tạo thư mục chỉ định của người dùng
+      Dir.mkdir path unless File.exist? path
+
+      threads = []
+      images.each do |curr_image| 
+          threads << Thread.new(curr_image){
+              open(curr_image[:link]) do |image|
+                  File.open("#{path}/".concat(curr_image[:link].split('/').last.to_s.split('?').first.to_s),"a+") do |file|
+                      file.write(image.read)              
+                      curr_image[:size] = image.size.to_s             
+                  end 
+              end
+
+          }
+      end
+      threads.each{|t| t.join}
     end
-  end
-end
 
+    def run(keyword, destination, max)
+
+      url = "https://www.123rf.com/stock-photo/dog.html?start=1&sti=mwk3081ovjk7062i9c|"
+      unparsed_page = open("#{url}/#{keyword}").read
+      parsed_page = Nokogiri::HTML(unparsed_page)
+      jobs = Array.new
+
+      list_imgs = parsed_page.css('div.mosaic-main-container') 
+      #list_imgs.count = 110 imges
+      #Ta sẽ crawl data từ web với số trang tối đa mà ta muốn crawl data
+
+      count_page  = 1  
+      #Tạo file dùng để ghi những thông tin của file
+      File.delete("images.xls") if File.file?("images.xls")
+      File.new("images.xls","a+")
+      File.open("images.xls","a") do |file|
+          file.write("NAME,URL,SIZE,EXTENSION")
+          file.write("\n")
+      end
+      
+
+
+      while count_page <= max*110
+          pagination_url ="https://www.123rf.com/stock-photo/#{keyword}.html?start=#{count_page}&sti=mwk3081ovjk7062i9c|"
+          pagination_unparsed_page = open(pagination_url).read
+          pagination_parsed_page = Nokogiri::HTML( pagination_unparsed_page  )
+          pagination_list_imgs =  pagination_parsed_page.css('div.mosaic-main-container')
+          puts "Page :#{count_page}"
+          puts pagination_url
+          puts " ..............."
+
+
+          #Ghi những thông tin về ảnh mà ta muốn download được lưu vào trong mảng jobs
+          pagination_list_imgs.each do |img_list|
+              job={
+                  name: img_list.css('img').attr('src').text.split('/').last.to_s,
+                  link: img_list.css('img').attr('src').text,
+                  size: 'nil',
+                  extension: img_list.css('img').attr('src').text.split('.').last.to_s.split('?').first.to_s
+      
+              }   
+              jobs << job  
+          end
+          downloadImages(jobs, destination)
+      
+
+          count_page+=110
+      end
+
+      #Ghi những thông tin ảnh vào file excel mà ta đã download về được từ mảng jobs
+      jobs.each do |curr_image| 
+          File.open("./images.xls","a+") do |file|
+              file.write(curr_image[:name].concat(",").concat(curr_image[:link]).concat(",").concat(curr_image[:size]).concat(",").concat(curr_image[:extension]))
+            file.write("\n")
+          end
+      end
+
+    end
+    end
+end

data/lib/tomosia_123rf_crawl/cli.rb

@@ -0,0 +1,14 @@
+require 'thor'
+require_relative '../tomosia_123rf_crawl'
+
+module Tomosia123rfCrawl
+  class Cli < Thor
+
+    desc "crawl KEYWORD", "enter KEYWORD to search"
+    option :destination
+    option :max
+    def crawl(keyword) # dung roi nay
+        Tomosia123rfCrawl::Crawler.new.run(keyword, options[:destination], options[:max].to_i)
+    end
+  end
+end

data/lib/tomosia_123rf_crawl/version.rb

@@ -1,3 +1,3 @@
 module Tomosia123rfCrawl
-  VERSION = "0.1.0"
+  VERSION = "0.1.9"
 end

data/tomosia_123rf_crawl.gemspec

@@ -3,26 +3,29 @@ require_relative 'lib/tomosia_123rf_crawl/version'
 Gem::Specification.new do |spec|
   spec.name          = "tomosia_123rf_crawl"
   spec.version       = Tomosia123rfCrawl::VERSION
-  spec.authors       = "MinhTuong"
-  spec.email         = "totkitano@gmail.com"
-  spec.summary       = %q{gem complete date 7/8/2020}
-  spec.description   = %q{gem crawl data to web.}
-  spec.homepage      = "https://github.com/MinhTuongbk1210/Gem_tomosia_123rf_crawl"
-  spec.license       = "MIT"
+  spec.authors       = ["Minh Tuong"]
+  spec.email         = ["tt.tuong.tran@tomosia.com"]
+
+  spec.summary       = %q{gem complete 7/8/2020.This is gem create}
+  spec.description   = %q{This is gem cool gem.}
+  spec.homepage      = "https://github.com/MinhTuongbk1210/gem_crawl_data"
   spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
 
   # spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
 
-  # spec.metadata["homepage_uri"] = spec.homepage
+   spec.metadata["homepage_uri"] = spec.homepage
   # spec.metadata["source_code_uri"] = "TODO: Put your gem's public repo URL here."
   # spec.metadata["changelog_uri"] = "TODO: Put your gem's CHANGELOG.md URL here."
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  end
+  spec.files         = `git ls-files`.split("\n")
   spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.executables   = ["tomosia_123rf_crawl"]
   spec.require_paths = ["lib"]
+  #spec.add_dependency 'thor'
+  spec.add_runtime_dependency('httparty', '0.18.1')
+  spec.add_runtime_dependency('nokogiri', '1.10.10')
+  spec.add_runtime_dependency('spreadsheet', '1.2.6')
+  spec.add_runtime_dependency('thor')
 end

metadata

@@ -1,35 +1,96 @@
 --- !ruby/object:Gem::Specification
 name: tomosia_123rf_crawl
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.9
 platform: ruby
 authors:
-- MinhTuong
+- Minh Tuong
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-08-07 00:00:00.000000000 Z
-dependencies: []
-description: gem crawl data to web.
-email: totkitano@gmail.com
-executables: []
+date: 2020-08-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.1
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.10.10
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.10.10
+- !ruby/object:Gem::Dependency
+  name: spreadsheet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.2.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.2.6
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This is gem cool gem.
+email:
+- tt.tuong.tran@tomosia.com
+executables:
+- tomosia_123rf_crawl
 extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
 - CODE_OF_CONDUCT.md
 - Gemfile
+- Gemfile.lock
 - README.md
 - Rakefile
 - bin/console
 - bin/setup
+- exe/tomosia_123rf_crawl
 - lib/tomosia_123rf_crawl.rb
+- lib/tomosia_123rf_crawl/cli.rb
 - lib/tomosia_123rf_crawl/version.rb
 - tomosia_123rf_crawl.gemspec
-homepage: https://github.com/MinhTuongbk1210/Gem_tomosia_123rf_crawl
-licenses:
-- MIT
-metadata: {}
+homepage: https://github.com/MinhTuongbk1210/gem_crawl_data
+licenses: []
+metadata:
+  homepage_uri: https://github.com/MinhTuongbk1210/gem_crawl_data
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -48,5 +109,5 @@ requirements: []
 rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
-summary: gem complete date 7/8/2020
+summary: gem complete 7/8/2020.This is gem create
 test_files: []