amajp_crawl 0.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 6911c11b2c19a7251e2ffce7548debf321b2b9376dfc3dcf6cd33188f24c49f3
+  data.tar.gz: 4c11bec39d6eaff44cbece13bd34cc7021e4c87fd5c3f9990b65c5bd30f1e927
+SHA512:
+  metadata.gz: b9c50a5a976072a3f6ef141db547767f5d7836c98f79238b8c5538b3b81188e8cb04bfc1ab611ba9ae2447aee2fc0659b1611db1804ed01754629c18f7d69f35
+  data.tar.gz: ff1edeb870edd2a4db9bc9177c553237e822dc5672bcdb354df73fe25a6d7a29d773e567c0c74bd1b1e5ecebd2f7af0aefc2ce170490fe7136a157111ea09f22

data/lib/amajp_crawl.rb

@@ -0,0 +1,121 @@
+require 'nokogiri'
+require 'open-uri'
+require 'csv'
+require 'thread'
+require 'pry'
+
+class Notice
+
+  def insert_keyword
+    "= = = = = = = = = = INSERT KEYWORD = = = = = = = = = = = = = = = ="
+  end
+
+  def insert_last_page
+    "= = = = = = = INSERT LAST PAGE = = = MAX IS 21 PAGE = = = = = = = "
+  end
+
+  def nil_keyword
+    "= = = = = = = = = = KEYWORD NILL = = = = = = = = = = = = = = = = ="
+  end
+
+  def not_availability_last_page
+    "= = = = = = = = = = LAST PAGE NOT AVAILABILITY = = = = = = = = = ="
+  end
+
+  def complete_get_asin
+    "= = = = = = = = = = COMPLETE GET DATA ASIN = = = = = = = = = = = ="
+  end
+
+  def insert_to_csv
+    "= = = = = = = = = = INSERT TO CSV FILE = = = = = = = = = = = = = ="
+  end
+
+  def complete
+    "= = = = = = = = = = COMPLETE = = = = = = = = = = = = = = = = = = ="
+  end
+
+  def loading
+    "== "
+  end
+
+  def error
+    "= = = = = = = = = = ERROR = = = TRY AGAIN LATER = = = = = = = = = "
+  end
+end
+
+class InsertData < Notice
+  attr_accessor :keyword, :last_page
+
+  def initialize
+    puts insert_keyword
+    @keyword = gets.chomp.to_s
+    puts insert_last_page
+    @last_page = gets.chomp.to_i
+  end
+end
+
+
+class GetdataAsin
+  attr_accessor :urls
+  
+  def initialize(insert_data)
+    $retries = 3
+    $completed = true
+    @urls = []
+    $total_asin = []
+    insert_data.last_page.times {|i| urls.push("https://www.amazon.co.jp/s?__mk_ja_JP=%E3%82%AB%E3%82%BF%E3%82%AB%E3%83%8A&i=aps&k=#{insert_data.keyword}&page=#{i+1}&ref=nb_sb_noss&url=search-alias%3Daps")}
+  end
+end
+
+insert_data = InsertData.new
+get_data = GetdataAsin.new(insert_data)
+
+begin
+  class Parser < Notice
+
+    def initialize(insert_data)
+      ## Function
+      insert_data.urls.each { |url|
+        raw_page = Nokogiri::HTML(open(url))
+        items = raw_page.css("#a-page > #search > .sg-row > .sg-col > .sg-col-inner > .rush-component > .s-result-list .s-result-item")
+        asin_array = []
+        items.each do |item|
+          asin = item.attr("data-asin")
+          unless asin == nil
+            asin_array << asin
+          end
+        end
+        $total_asin = $total_asin + asin_array
+        print loading
+      }
+      puts complete_get_asin
+      puts insert_to_csv
+    end
+  end
+
+parser_data = Parser.new(get_data)
+
+rescue => exception
+  puts exception
+  puts "Wait for 3 minutes and Retry"
+  sleep 180 # 3 minutes
+  $retries -= 1
+  retry if $retries > 0 
+  $completed = false
+  puts "#{exception}, TRY AGAIN LATER" 
+end
+
+class InsertCsv < Notice
+  def initialize(insert_data)
+    CSV.open("#{insert_data.keyword}_asin_amazonjp.csv", 'wb') do |csv|
+      $total_asin.each do |item|
+        csv << [item, "\n"]
+      end
+    end
+    current_directory = Dir.pwd + "/" + "#{insert_data.keyword}_asin_amazonjp.csv"
+    return puts error if $completed == false
+    puts complete
+    puts "CHECK #{insert_data.keyword} ASIN AT  #{current_directory}"
+  end
+end
+InsertCsv.new(insert_data)

metadata

@@ -0,0 +1,43 @@
+--- !ruby/object:Gem::Specification
+name: amajp_crawl
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+platform: ruby
+authors:
+- Le Hoang Lan
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-07-07 00:00:00.000000000 Z
+dependencies: []
+description: 
+email: lehoanglan97k31@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/amajp_crawl.rb
+homepage: http://rubygems.org/gems/amajp_crawl
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Crawl AsinID any production from Amazonjp via Terminal
+test_files: []