tomosia_amanaplus_crawl 0.1.3 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1daa69b29114aa1cd35172183b838b1dec522be2cc796449f6cfcb93c279a7de
-  data.tar.gz: 1f656912c20d6e924ac40a0bc87f60b5e3c01762ccc2d2f4c72c202e23acfad0
+  metadata.gz: 793268d3d7e8d3c1f3fd17ca9f1283839dda2a20dfb1a8769c65fd6d25bf0236
+  data.tar.gz: 115062f473eb2ca55c9ceea17265b73827b53feb548a48861b808b9baa64447f
 SHA512:
-  metadata.gz: 86f3eeed3acbe50e039ac50b1359f9b8894354d966dcf84321465c1584789bb77444acc40d608380e53b7873c265699f5e5501f44883b0178ab95baf61b87ee6
-  data.tar.gz: 5556e3ea60053e307dc1ef5f08412220046cf07fe63ad018d99d4f82a9e756bc66a9539e3c0c190ac0de8f6a3e6cdeb94b783bdb3f2c4730300bac5e70fb9973
+  metadata.gz: 83ee2ae24471817f907373dd2130fd54726786eb2fca7732e9fbe10ec841b4b5cbfc3fa4925753d725eb23362c330cec055a8b43eb8f0321053e7ce497f77930
+  data.tar.gz: 5b92ea21908fa8288a2fa26af61c8c198b525f0de6f6aba7766e89931f064edd0bd37059d2fc205b8090bf0aac2825865fa5fe9ce8226dd7003efb74f55ecf2b

data/Gemfile

@@ -5,8 +5,3 @@ gemspec
 
 gem "rake", "~> 12.0"
 gem "rspec", "~> 3.0"
-
-gem 'httparty'
-gem 'nokogiri'
-gem 'spreadsheet'
-gem 'byebug'

data/Gemfile.lock

@@ -0,0 +1,50 @@
+PATH
+  remote: .
+  specs:
+    tomosia_amanaplus_crawl (0.1.5)
+      httparty (= 0.18.1)
+      nokogiri (= 1.10.10)
+      spreadsheet (= 1.2.6)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.4.4)
+    httparty (0.18.1)
+      mime-types (~> 3.0)
+      multi_xml (>= 0.5.2)
+    mime-types (3.3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2020.0512)
+    mini_portile2 (2.4.0)
+    multi_xml (0.6.0)
+    nokogiri (1.10.10)
+      mini_portile2 (~> 2.4.0)
+    rake (12.3.3)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+    ruby-ole (1.2.12.2)
+    spreadsheet (1.2.6)
+      ruby-ole (>= 1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rake (~> 12.0)
+  rspec (~> 3.0)
+  tomosia_amanaplus_crawl!
+
+BUNDLED WITH
+   2.1.4

data/README.md

@@ -22,7 +22,13 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+```ruby
+    require 'tomosia_amanaplus_crawl'
+    TomosiaAmanaplusCrawl::Crawler.new.run(keyword, path, max)
+```
+    keyword: hoian, danang, ...
+    path: './', '/desktop/', ...
+    max: số lượng ảnh muốn lấy về. Nếu max lớn hơn tổng số ảnh các page thì vẫn lấy hết tất cả ảnh
 
 ## Development
 

data/lib/tomosia_amanaplus_crawl.rb

@@ -1,6 +1,10 @@
 require "tomosia_amanaplus_crawl/version"
 
 module TomosiaAmanaplusCrawl
+  def self.yeuNgucLep
+    puts "Yeu chi My nhieu lam"
+  end
+
   require 'nokogiri'
   require 'httparty'
   require 'open-uri'
@@ -10,21 +14,29 @@ module TomosiaAmanaplusCrawl
   class Crawler
     URL = "https://plus.amanaimages.com/items/search/"
 
-    def run(keyword, destination)
+    def run(keyword, destination, max)
       unparsed_page = HTTParty.get("#{URL}/#{keyword}")
       parsed_page = Nokogiri::HTML(unparsed_page)
 
       pages = parsed_page.css("div.c-paginate__nums").css('a').last.text.to_i # tổng số page
       images_listings = parsed_page.css("div.p-search-result__body") # danh sách các thẻ div chứa image
+    
+      # lấy tổng số image
+      total = parsed_page.css("h1.p-search-result__ttl").text.split(' ').first
+      total = total[11..(total.length - 1)].chop.chop.chop.sub(',', '').to_i
+      if max > total # nếu max lớn hơn total thì max = total => vẫn lấy hết
+        max = total
+      end
 
-      images = getPaginationImages(images_listings, pages, keyword)
+      images = getPaginationImages(images_listings, pages, keyword, max)
       downloadImages(images, destination)
       writeToExcel(images, destination)
     end
 
-    def getPaginationImages(images_listings, pages, keyword)  # lấy tất cả image của các page cộng lại
+    def getPaginationImages(images_listings, pages, keyword, max)  # lấy tất cả image của các page cộng lại
       images = Array.new
       curr_page = 1
+      curr_index = 1
       while curr_page <= pages
         puts "Crawling page #{curr_page}..........."
     
@@ -33,6 +45,10 @@ module TomosiaAmanaplusCrawl
         pagination_images_listings = pagination_parsed_page.css("div.p-item-thumb")
     
         pagination_images_listings.each do |img|
+          if curr_index > max
+            return images
+          end
+
           src = img.css('img').attr('data-src').nil? == true ? img.css('img').attr('src') : img.css('img').attr('data-src')
           current_image = {
             title: img.css('a')[1].attr('title'),
@@ -41,6 +57,7 @@ module TomosiaAmanaplusCrawl
             extension: ".#{src.to_s.split('.').last}"
           }
           images << current_image
+          curr_index += 1
         end
     
         curr_page += 1

data/lib/tomosia_amanaplus_crawl/version.rb

@@ -1,3 +1,3 @@
 module TomosiaAmanaplusCrawl
-  VERSION = "0.1.3"
+  VERSION = "0.1.8"
 end

data/tomosia_amanaplus_crawl.gemspec

@@ -14,4 +14,8 @@ Gem::Specification.new do |spec|
   end
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency('httparty', '0.18.1')
+  spec.add_runtime_dependency('nokogiri', '1.10.10')
+  spec.add_runtime_dependency('spreadsheet', '1.2.6')
 end

metadata

@@ -1,15 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: tomosia_amanaplus_crawl
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.8
 platform: ruby
 authors:
 - Nhat Huy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-08-06 00:00:00.000000000 Z
-dependencies: []
+date: 2020-08-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.1
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.10.10
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.10.10
+- !ruby/object:Gem::Dependency
+  name: spreadsheet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.2.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.2.6
 description: 
 email: 
 executables: []
@@ -21,6 +63,7 @@ files:
 - ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile