tomosia_amanaplus_crawl 0.1.0 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 12228835df204c09456c99bdd9ff297d72696872ed2b90af6cbef34b0e55b293
-  data.tar.gz: e5e2390aa64bb8e32ef48ac9f4f3ff51d8baa20a01da42126b1f26efb1902eb3
+  metadata.gz: 7e09b5361e7b84f880485d64480eca862a19dcbd32c0cf4a0bab3160c3b5ab11
+  data.tar.gz: fc69ac8dcb7e3ba4c74412bb4760ef2cd996831d572eb63f184624330d8789a8
 SHA512:
-  metadata.gz: e160b67d73d2bafadfa57c3e492868f10bfd3ce6e8232a065faa80d9563f9d333ac9a16792c86f36c9cc0e80b344fc0c8c18e1076cf26803fb73478b4d8fe5d2
-  data.tar.gz: 8b415f28cf91e873ebad4cdb94faca4cc231db680436294e7b7a01b31025bd9f1f60a404a420fac1897078b62693af1306cf101b1676c1df8bdf695bf35ac9d3
+  metadata.gz: 8ef2cbc7fa812b63ff04e136dcc5ff479fede20b7489485b83ce5dc84c5642fe3eba846bfc8a14c335b7c9271acf4bbcf2a6e0d4e2e557302488359306e820f4
+  data.tar.gz: 414a5491c0227c9f43830a11384e90b55305b0cdcb14d1b963c60c36aa278b748631296ad4bee837ed05184c490df8f747a0cc239025c6bd12d831ddb3ecb704

data/Gemfile

@@ -5,7 +5,3 @@ gemspec
 
 gem "rake", "~> 12.0"
 gem "rspec", "~> 3.0"
-gem 'httparty'
-gem 'nokogiri'
-gem 'spreadsheet'
-gem 'byebug'

data/Gemfile.lock

@@ -0,0 +1,50 @@
+PATH
+  remote: .
+  specs:
+    tomosia_amanaplus_crawl (0.1.5)
+      httparty (= 0.18.1)
+      nokogiri (= 1.10.10)
+      spreadsheet (= 1.2.6)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.4.4)
+    httparty (0.18.1)
+      mime-types (~> 3.0)
+      multi_xml (>= 0.5.2)
+    mime-types (3.3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2020.0512)
+    mini_portile2 (2.4.0)
+    multi_xml (0.6.0)
+    nokogiri (1.10.10)
+      mini_portile2 (~> 2.4.0)
+    rake (12.3.3)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+    ruby-ole (1.2.12.2)
+    spreadsheet (1.2.6)
+      ruby-ole (>= 1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rake (~> 12.0)
+  rspec (~> 3.0)
+  tomosia_amanaplus_crawl!
+
+BUNDLED WITH
+   2.1.4

data/README.md

@@ -10,6 +10,9 @@ Add this line to your application's Gemfile:
 
 ```ruby
 gem 'tomosia_amanaplus_crawl'
+gem 'httparty'
+gem 'nokogiri'
+gem 'spreadsheet'
 ```
 
 And then execute:
@@ -22,7 +25,13 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+```ruby
+    require 'tomosia_amanaplus_crawl'
+    TomosiaAmanaplusCrawl::Crawler.new.run(keyword, path, max)
+```
+    keyword: hoian, danang, ...
+    path: './', '/desktop/', ...
+    max: số lượng ảnh muốn lấy về. Nếu max lớn hơn tổng số ảnh các page thì vẫn lấy hết tất cả ảnh
 
 ## Development
 

data/lib/tomosia_amanaplus_crawl.rb

@@ -1,30 +1,38 @@
 require "tomosia_amanaplus_crawl/version"
-require 'nokogiri'
-require 'httparty'
-require 'open-uri'
-require 'fileutils'
-require 'spreadsheet'
 
 module TomosiaAmanaplusCrawl
+  require 'nokogiri'
+  require 'httparty'
+  require 'open-uri'
+  require 'fileutils'
+  require 'spreadsheet'
+
   class Crawler
     URL = "https://plus.amanaimages.com/items/search/"
 
-    def run(keyword, destination)
+    def run(keyword, destination, max)
       unparsed_page = HTTParty.get("#{URL}/#{keyword}")
       parsed_page = Nokogiri::HTML(unparsed_page)
 
       pages = parsed_page.css("div.c-paginate__nums").css('a').last.text.to_i # tổng số page
       images_listings = parsed_page.css("div.p-search-result__body") # danh sách các thẻ div chứa image
+    
+      # lấy tổng số image
+      total = parsed_page.css("h1.p-search-result__ttl").text.split(' ').first
+      total = total[11..(total.length - 1)].chop.chop.chop.sub(',', '').to_i
+      if max > total # nếu max lớn hơn total thì max = total => vẫn lấy hết
+        max = total
+      end
 
-      images = getPaginationImages(images_listings, pages, keyword)
+      images = getPaginationImages(images_listings, pages, keyword, max)
       downloadImages(images, destination)
       writeToExcel(images, destination)
     end
 
-    def getPaginationImages(images_listings, pages, keyword)  # lấy tất cả image của các page cộng lại
+    def getPaginationImages(images_listings, pages, keyword, max)  # lấy tất cả image của các page cộng lại
       images = Array.new
-      i = 0
       curr_page = 1
+      curr_index = 1
       while curr_page <= pages
         puts "Crawling page #{curr_page}..........."
     
@@ -33,6 +41,10 @@ module TomosiaAmanaplusCrawl
         pagination_images_listings = pagination_parsed_page.css("div.p-item-thumb")
     
         pagination_images_listings.each do |img|
+          if curr_index > max
+            return images
+          end
+
           src = img.css('img').attr('data-src').nil? == true ? img.css('img').attr('src') : img.css('img').attr('data-src')
           current_image = {
             title: img.css('a')[1].attr('title'),
@@ -41,7 +53,7 @@ module TomosiaAmanaplusCrawl
             extension: ".#{src.to_s.split('.').last}"
           }
           images << current_image
-          # puts "#{i += 1}: #{src}"
+          curr_index += 1
         end
     
         curr_page += 1

data/lib/tomosia_amanaplus_crawl/version.rb

@@ -1,3 +1,3 @@
 module TomosiaAmanaplusCrawl
-  VERSION = "0.1.0"
+  VERSION = "0.1.5"
 end

data/tomosia_amanaplus_crawl.gemspec

@@ -4,26 +4,18 @@ Gem::Specification.new do |spec|
   spec.name          = "tomosia_amanaplus_crawl"
   spec.version       = TomosiaAmanaplusCrawl::VERSION
   spec.authors       = "Nhat Huy"
-  spec.email         = "nhathuych@tomosia.com"
 
   spec.summary       = %q{tomosia_amanaplus_crawl demo project crawl du lieu.}
-  spec.description   = %q{tomosia_amanaplus_crawl demo project crawl du lieu.}
   spec.homepage      = "https://github.com/tthuydang/tomosia_amanaplus_crawl"
-  spec.license       = "MIT"
   spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
 
-  # spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
-
-  # spec.metadata["homepage_uri"] = spec.homepage
-  # spec.metadata["source_code_uri"] = "TODO: Put your gem's public repo URL here."
-  # spec.metadata["changelog_uri"] = "TODO: Put your gem's CHANGELOG.md URL here."
-
-  # Specify which files should be added to the gem when it is released.
-  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
-  spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency('httparty', '0.18.1')
+  spec.add_runtime_dependency('nokogiri', '1.10.10')
+  spec.add_runtime_dependency('spreadsheet', '1.2.6')
 end

metadata

@@ -1,17 +1,59 @@
 --- !ruby/object:Gem::Specification
 name: tomosia_amanaplus_crawl
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.5
 platform: ruby
 authors:
 - Nhat Huy
 autorequire: 
-bindir: exe
+bindir: bin
 cert_chain: []
-date: 2020-08-04 00:00:00.000000000 Z
-dependencies: []
-description: tomosia_amanaplus_crawl demo project crawl du lieu.
-email: nhathuych@tomosia.com
+date: 2020-08-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.1
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.10.10
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.10.10
+- !ruby/object:Gem::Dependency
+  name: spreadsheet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.2.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.2.6
+description: 
+email: 
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -21,6 +63,7 @@ files:
 - ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -30,8 +73,7 @@ files:
 - lib/tomosia_amanaplus_crawl/version.rb
 - tomosia_amanaplus_crawl.gemspec
 homepage: https://github.com/tthuydang/tomosia_amanaplus_crawl
-licenses:
-- MIT
+licenses: []
 metadata: {}
 post_install_message: 
 rdoc_options: []