tomosia_amanaplus_crawl 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1daa69b29114aa1cd35172183b838b1dec522be2cc796449f6cfcb93c279a7de
-  data.tar.gz: 1f656912c20d6e924ac40a0bc87f60b5e3c01762ccc2d2f4c72c202e23acfad0
+  metadata.gz: 2832ce228e9bc2ebcce58c1dde236cc0f6bfa4c789654e976cb66defe368fb39
+  data.tar.gz: 790070002318d2d3c3727b9fd3986b655624579d3130e32e0ba6c88bd65485cd
 SHA512:
-  metadata.gz: 86f3eeed3acbe50e039ac50b1359f9b8894354d966dcf84321465c1584789bb77444acc40d608380e53b7873c265699f5e5501f44883b0178ab95baf61b87ee6
-  data.tar.gz: 5556e3ea60053e307dc1ef5f08412220046cf07fe63ad018d99d4f82a9e756bc66a9539e3c0c190ac0de8f6a3e6cdeb94b783bdb3f2c4730300bac5e70fb9973
+  metadata.gz: 8b174ec110e8933aae815abef670ae0a33ae69077b98268522aaf83a52702f7d3c303cc9bbf73d5e8ec7d27a9cb2deb1cd1d05e6edfc9b5ecff8fa94b49af4d9
+  data.tar.gz: badfef5587e0b7989a313caf4490f59efba61d9e743f16126ae1bd7915abd7d878be1467bbcef967c4968c590c7c4383f542c0a8a51f5bf07783759ae4b216da

data/Gemfile

@@ -9,4 +9,3 @@ gem "rspec", "~> 3.0"
 gem 'httparty'
 gem 'nokogiri'
 gem 'spreadsheet'
-gem 'byebug'

data/README.md

@@ -10,6 +10,9 @@ Add this line to your application's Gemfile:
 
 ```ruby
 gem 'tomosia_amanaplus_crawl'
+gem 'httparty'
+gem 'nokogiri'
+gem 'spreadsheet'
 ```
 
 And then execute:
@@ -22,7 +25,13 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+```ruby
+    require 'tomosia_amanaplus_crawl'
+    TomosiaAmanaplusCrawl::Crawler.new.run(keyword, path, max)
+```
+    keyword: hoian, danang, ...
+    path: './', '/desktop/', ...
+    max: số lượng ảnh muốn lấy về. Nếu max lớn hơn tổng số ảnh các page thì vẫn lấy hết tất cả ảnh
 
 ## Development
 

data/lib/tomosia_amanaplus_crawl.rb

@@ -10,21 +10,29 @@ module TomosiaAmanaplusCrawl
   class Crawler
     URL = "https://plus.amanaimages.com/items/search/"
 
-    def run(keyword, destination)
+    def run(keyword, destination, max)
       unparsed_page = HTTParty.get("#{URL}/#{keyword}")
       parsed_page = Nokogiri::HTML(unparsed_page)
 
       pages = parsed_page.css("div.c-paginate__nums").css('a').last.text.to_i # tổng số page
       images_listings = parsed_page.css("div.p-search-result__body") # danh sách các thẻ div chứa image
+    
+      # lấy tổng số image
+      total = parsed_page.css("h1.p-search-result__ttl").text.split(' ').first
+      total = total[11..(total.length - 1)].chop.chop.chop.sub(',', '').to_i
+      if max > total # nếu max lớn hơn total thì max = total => vẫn lấy hết
+        max = total
+      end
 
-      images = getPaginationImages(images_listings, pages, keyword)
+      images = getPaginationImages(images_listings, pages, keyword, max)
       downloadImages(images, destination)
       writeToExcel(images, destination)
     end
 
-    def getPaginationImages(images_listings, pages, keyword)  # lấy tất cả image của các page cộng lại
+    def getPaginationImages(images_listings, pages, keyword, max)  # lấy tất cả image của các page cộng lại
       images = Array.new
       curr_page = 1
+      curr_index = 1
       while curr_page <= pages
         puts "Crawling page #{curr_page}..........."
     
@@ -33,6 +41,10 @@ module TomosiaAmanaplusCrawl
         pagination_images_listings = pagination_parsed_page.css("div.p-item-thumb")
     
         pagination_images_listings.each do |img|
+          if curr_index > max
+            return images
+          end
+
           src = img.css('img').attr('data-src').nil? == true ? img.css('img').attr('src') : img.css('img').attr('data-src')
           current_image = {
             title: img.css('a')[1].attr('title'),
@@ -41,6 +53,7 @@ module TomosiaAmanaplusCrawl
             extension: ".#{src.to_s.split('.').last}"
           }
           images << current_image
+          curr_index += 1
         end
     
         curr_page += 1

data/lib/tomosia_amanaplus_crawl/version.rb

@@ -1,3 +1,3 @@
 module TomosiaAmanaplusCrawl
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tomosia_amanaplus_crawl
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Nhat Huy